tardis-machine 18.0.0 → 18.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/benchmark.js +35 -14
  2. package/bin/tardis-machine.js +0 -1
  3. package/dist/tardismachine.d.ts.map +1 -1
  4. package/dist/tardismachine.js +20 -55
  5. package/dist/tardismachine.js.map +1 -1
  6. package/dist/ws/replay.d.ts +2 -2
  7. package/dist/ws/replay.d.ts.map +1 -1
  8. package/dist/ws/replay.js +11 -5
  9. package/dist/ws/replay.js.map +1 -1
  10. package/dist/ws/replaynormalized.d.ts +2 -2
  11. package/dist/ws/replaynormalized.d.ts.map +1 -1
  12. package/dist/ws/replaynormalized.js +7 -5
  13. package/dist/ws/replaynormalized.js.map +1 -1
  14. package/dist/ws/server.d.ts +20 -0
  15. package/dist/ws/server.d.ts.map +1 -0
  16. package/dist/ws/server.js +154 -0
  17. package/dist/ws/server.js.map +1 -0
  18. package/dist/ws/streamnormalized.d.ts +2 -2
  19. package/dist/ws/streamnormalized.d.ts.map +1 -1
  20. package/dist/ws/streamnormalized.js +7 -5
  21. package/dist/ws/streamnormalized.js.map +1 -1
  22. package/dist/ws/subscriptionsmappers.d.ts +1 -1
  23. package/dist/ws/subscriptionsmappers.d.ts.map +1 -1
  24. package/node_modules/tardis-dev/README.md +1 -1
  25. package/node_modules/tardis-dev/dist/debug.d.ts +12 -2
  26. package/node_modules/tardis-dev/dist/debug.d.ts.map +1 -1
  27. package/node_modules/tardis-dev/dist/debug.js +2 -1
  28. package/node_modules/tardis-dev/dist/debug.js.map +1 -1
  29. package/node_modules/tardis-dev/dist/downloaddatasets.d.ts.map +1 -1
  30. package/node_modules/tardis-dev/dist/downloaddatasets.js +3 -1
  31. package/node_modules/tardis-dev/dist/downloaddatasets.js.map +1 -1
  32. package/node_modules/tardis-dev/dist/handy.d.ts +5 -2
  33. package/node_modules/tardis-dev/dist/handy.d.ts.map +1 -1
  34. package/node_modules/tardis-dev/dist/handy.js +77 -90
  35. package/node_modules/tardis-dev/dist/handy.js.map +1 -1
  36. package/node_modules/tardis-dev/dist/mappers/hyperliquid.d.ts +1 -1
  37. package/node_modules/tardis-dev/dist/mappers/hyperliquid.d.ts.map +1 -1
  38. package/node_modules/tardis-dev/dist/mappers/hyperliquid.js +15 -4
  39. package/node_modules/tardis-dev/dist/mappers/hyperliquid.js.map +1 -1
  40. package/node_modules/tardis-dev/dist/mappers/index.d.ts +1 -1
  41. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.d.ts +0 -7
  42. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.d.ts.map +1 -1
  43. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.js +2 -70
  44. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.js.map +1 -1
  45. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.d.ts +43 -0
  46. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.d.ts.map +1 -0
  47. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.js +294 -0
  48. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.js.map +1 -0
  49. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.d.ts +3 -3
  50. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
  51. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.js +3 -3
  52. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.js.map +1 -1
  53. package/node_modules/tardis-dev/dist/replay.d.ts.map +1 -1
  54. package/node_modules/tardis-dev/dist/replay.js +6 -4
  55. package/node_modules/tardis-dev/dist/replay.js.map +1 -1
  56. package/node_modules/tardis-dev/dist/worker.d.ts.map +1 -1
  57. package/node_modules/tardis-dev/dist/worker.js +7 -4
  58. package/node_modules/tardis-dev/dist/worker.js.map +1 -1
  59. package/node_modules/tardis-dev/dist/zstd.d.ts +3 -0
  60. package/node_modules/tardis-dev/dist/zstd.d.ts.map +1 -0
  61. package/node_modules/tardis-dev/dist/zstd.js +122 -0
  62. package/node_modules/tardis-dev/dist/zstd.js.map +1 -0
  63. package/node_modules/tardis-dev/package.json +3 -7
  64. package/node_modules/tardis-dev/src/debug.ts +16 -1
  65. package/node_modules/tardis-dev/src/downloaddatasets.ts +3 -1
  66. package/node_modules/tardis-dev/src/external-modules.d.ts +5 -0
  67. package/node_modules/tardis-dev/src/handy.ts +100 -93
  68. package/node_modules/tardis-dev/src/mappers/hyperliquid.ts +19 -5
  69. package/node_modules/tardis-dev/src/realtimefeeds/mexc.ts +2 -72
  70. package/node_modules/tardis-dev/src/realtimefeeds/mexcprotobuf.ts +364 -0
  71. package/node_modules/tardis-dev/src/realtimefeeds/realtimefeed.ts +5 -5
  72. package/node_modules/tardis-dev/src/replay.ts +6 -4
  73. package/node_modules/tardis-dev/src/worker.ts +7 -4
  74. package/node_modules/tardis-dev/src/zstd.ts +139 -0
  75. package/package.json +9 -35
  76. package/src/tardismachine.ts +25 -66
  77. package/src/ws/replay.ts +13 -8
  78. package/src/ws/replaynormalized.ts +8 -6
  79. package/src/ws/server.ts +173 -0
  80. package/src/ws/streamnormalized.ts +8 -6
  81. package/src/ws/subscriptionsmappers.ts +1 -1
  82. package/node_modules/follow-redirects/LICENSE +0 -18
  83. package/node_modules/follow-redirects/README.md +0 -157
  84. package/node_modules/follow-redirects/debug.js +0 -15
  85. package/node_modules/follow-redirects/http.js +0 -1
  86. package/node_modules/follow-redirects/https.js +0 -1
  87. package/node_modules/follow-redirects/index.js +0 -709
  88. package/node_modules/follow-redirects/package.json +0 -58
  89. package/node_modules/long/LICENSE +0 -202
  90. package/node_modules/long/README.md +0 -286
  91. package/node_modules/long/index.d.ts +0 -2
  92. package/node_modules/long/index.js +0 -1581
  93. package/node_modules/long/package.json +0 -58
  94. package/node_modules/long/types.d.ts +0 -474
  95. package/node_modules/long/umd/index.d.ts +0 -3
  96. package/node_modules/long/umd/index.js +0 -1622
  97. package/node_modules/long/umd/package.json +0 -3
  98. package/node_modules/long/umd/types.d.ts +0 -474
  99. package/node_modules/protobufjs/LICENSE +0 -39
  100. package/node_modules/protobufjs/README.md +0 -440
  101. package/node_modules/protobufjs/dist/light/protobuf.js +0 -9276
  102. package/node_modules/protobufjs/dist/light/protobuf.js.map +0 -1
  103. package/node_modules/protobufjs/dist/light/protobuf.min.js +0 -8
  104. package/node_modules/protobufjs/dist/light/protobuf.min.js.map +0 -1
  105. package/node_modules/protobufjs/dist/minimal/protobuf.js +0 -3556
  106. package/node_modules/protobufjs/dist/minimal/protobuf.js.map +0 -1
  107. package/node_modules/protobufjs/dist/minimal/protobuf.min.js +0 -8
  108. package/node_modules/protobufjs/dist/minimal/protobuf.min.js.map +0 -1
  109. package/node_modules/protobufjs/dist/protobuf.js +0 -11233
  110. package/node_modules/protobufjs/dist/protobuf.js.map +0 -1
  111. package/node_modules/protobufjs/dist/protobuf.min.js +0 -8
  112. package/node_modules/protobufjs/dist/protobuf.min.js.map +0 -1
  113. package/node_modules/protobufjs/ext/README.md +0 -70
  114. package/node_modules/protobufjs/ext/debug/README.md +0 -4
  115. package/node_modules/protobufjs/ext/debug/index.js +0 -71
  116. package/node_modules/protobufjs/ext/descriptor/README.md +0 -5
  117. package/node_modules/protobufjs/ext/descriptor/index.d.ts +0 -2
  118. package/node_modules/protobufjs/ext/descriptor/index.js +0 -2
  119. package/node_modules/protobufjs/ext/descriptor.d.ts +0 -87
  120. package/node_modules/protobufjs/ext/descriptor.generated.d.ts +0 -417
  121. package/node_modules/protobufjs/ext/descriptor.js +0 -1356
  122. package/node_modules/protobufjs/ext/protojson.LICENSE +0 -201
  123. package/node_modules/protobufjs/ext/protojson.d.ts +0 -20
  124. package/node_modules/protobufjs/ext/protojson.generated.d.ts +0 -49
  125. package/node_modules/protobufjs/ext/protojson.js +0 -951
  126. package/node_modules/protobufjs/ext/textformat.d.ts +0 -19
  127. package/node_modules/protobufjs/ext/textformat.generated.d.ts +0 -33
  128. package/node_modules/protobufjs/ext/textformat.js +0 -1276
  129. package/node_modules/protobufjs/google/LICENSE +0 -27
  130. package/node_modules/protobufjs/google/README.md +0 -1
  131. package/node_modules/protobufjs/google/api/annotations.json +0 -85
  132. package/node_modules/protobufjs/google/api/annotations.proto +0 -11
  133. package/node_modules/protobufjs/google/api/http.json +0 -87
  134. package/node_modules/protobufjs/google/api/http.proto +0 -31
  135. package/node_modules/protobufjs/google/protobuf/api.json +0 -124
  136. package/node_modules/protobufjs/google/protobuf/api.proto +0 -34
  137. package/node_modules/protobufjs/google/protobuf/compiler/plugin.json +0 -126
  138. package/node_modules/protobufjs/google/protobuf/compiler/plugin.proto +0 -47
  139. package/node_modules/protobufjs/google/protobuf/descriptor.json +0 -1465
  140. package/node_modules/protobufjs/google/protobuf/descriptor.proto +0 -527
  141. package/node_modules/protobufjs/google/protobuf/source_context.json +0 -21
  142. package/node_modules/protobufjs/google/protobuf/source_context.proto +0 -7
  143. package/node_modules/protobufjs/google/protobuf/type.json +0 -209
  144. package/node_modules/protobufjs/google/protobuf/type.proto +0 -89
  145. package/node_modules/protobufjs/index.d.ts +0 -3203
  146. package/node_modules/protobufjs/index.js +0 -4
  147. package/node_modules/protobufjs/light.d.ts +0 -2
  148. package/node_modules/protobufjs/light.js +0 -4
  149. package/node_modules/protobufjs/minimal.d.ts +0 -2
  150. package/node_modules/protobufjs/minimal.js +0 -4
  151. package/node_modules/protobufjs/package.json +0 -93
  152. package/node_modules/protobufjs/src/common.js +0 -405
  153. package/node_modules/protobufjs/src/converter.js +0 -352
  154. package/node_modules/protobufjs/src/decoder.js +0 -279
  155. package/node_modules/protobufjs/src/encoder.js +0 -123
  156. package/node_modules/protobufjs/src/enum.js +0 -239
  157. package/node_modules/protobufjs/src/field.js +0 -505
  158. package/node_modules/protobufjs/src/index-light.js +0 -104
  159. package/node_modules/protobufjs/src/index-minimal.js +0 -35
  160. package/node_modules/protobufjs/src/index.js +0 -12
  161. package/node_modules/protobufjs/src/mapfield.js +0 -144
  162. package/node_modules/protobufjs/src/message.js +0 -137
  163. package/node_modules/protobufjs/src/method.js +0 -183
  164. package/node_modules/protobufjs/src/namespace.js +0 -574
  165. package/node_modules/protobufjs/src/object.js +0 -392
  166. package/node_modules/protobufjs/src/oneof.js +0 -233
  167. package/node_modules/protobufjs/src/parse.js +0 -1111
  168. package/node_modules/protobufjs/src/reader.js +0 -788
  169. package/node_modules/protobufjs/src/reader_buffer.js +0 -77
  170. package/node_modules/protobufjs/src/root.js +0 -424
  171. package/node_modules/protobufjs/src/roots.js +0 -18
  172. package/node_modules/protobufjs/src/rpc/service.js +0 -155
  173. package/node_modules/protobufjs/src/rpc.js +0 -36
  174. package/node_modules/protobufjs/src/service.js +0 -206
  175. package/node_modules/protobufjs/src/tokenize.js +0 -421
  176. package/node_modules/protobufjs/src/type.js +0 -676
  177. package/node_modules/protobufjs/src/types.js +0 -196
  178. package/node_modules/protobufjs/src/typescript.js +0 -25
  179. package/node_modules/protobufjs/src/util/aspromise.d.ts +0 -13
  180. package/node_modules/protobufjs/src/util/aspromise.js +0 -52
  181. package/node_modules/protobufjs/src/util/base64.d.ts +0 -32
  182. package/node_modules/protobufjs/src/util/base64.js +0 -146
  183. package/node_modules/protobufjs/src/util/codegen.d.ts +0 -31
  184. package/node_modules/protobufjs/src/util/codegen.js +0 -118
  185. package/node_modules/protobufjs/src/util/eventemitter.d.ts +0 -45
  186. package/node_modules/protobufjs/src/util/eventemitter.js +0 -86
  187. package/node_modules/protobufjs/src/util/fetch.d.ts +0 -56
  188. package/node_modules/protobufjs/src/util/fetch.js +0 -112
  189. package/node_modules/protobufjs/src/util/float.d.ts +0 -83
  190. package/node_modules/protobufjs/src/util/float.js +0 -335
  191. package/node_modules/protobufjs/src/util/fs.js +0 -11
  192. package/node_modules/protobufjs/src/util/longbits.js +0 -204
  193. package/node_modules/protobufjs/src/util/minimal.js +0 -511
  194. package/node_modules/protobufjs/src/util/path.d.ts +0 -22
  195. package/node_modules/protobufjs/src/util/path.js +0 -100
  196. package/node_modules/protobufjs/src/util/patterns.js +0 -7
  197. package/node_modules/protobufjs/src/util/pool.d.ts +0 -32
  198. package/node_modules/protobufjs/src/util/pool.js +0 -48
  199. package/node_modules/protobufjs/src/util/utf8.d.ts +0 -24
  200. package/node_modules/protobufjs/src/util/utf8.js +0 -190
  201. package/node_modules/protobufjs/src/util.js +0 -264
  202. package/node_modules/protobufjs/src/verifier.js +0 -185
  203. package/node_modules/protobufjs/src/wrappers.js +0 -106
  204. package/node_modules/protobufjs/src/writer.js +0 -700
  205. package/node_modules/protobufjs/src/writer_buffer.js +0 -100
  206. package/node_modules/protobufjs/tsconfig.json +0 -6
  207. package/node_modules/uWebSockets.js/ESM_wrapper.mjs +0 -53
  208. package/node_modules/uWebSockets.js/LICENSE +0 -201
  209. package/node_modules/uWebSockets.js/README.md +0 -6
  210. package/node_modules/uWebSockets.js/index.d.ts +0 -468
  211. package/node_modules/uWebSockets.js/package.json +0 -12
  212. package/node_modules/uWebSockets.js/source_commit +0 -1
  213. package/node_modules/uWebSockets.js/uws.js +0 -76
  214. package/node_modules/uWebSockets.js/uws_darwin_arm64_127.node +0 -0
  215. package/node_modules/uWebSockets.js/uws_darwin_arm64_137.node +0 -0
  216. package/node_modules/uWebSockets.js/uws_darwin_arm64_147.node +0 -0
  217. package/node_modules/uWebSockets.js/uws_darwin_x64_127.node +0 -0
  218. package/node_modules/uWebSockets.js/uws_darwin_x64_137.node +0 -0
  219. package/node_modules/uWebSockets.js/uws_darwin_x64_147.node +0 -0
  220. package/node_modules/uWebSockets.js/uws_linux_arm64_127.node +0 -0
  221. package/node_modules/uWebSockets.js/uws_linux_arm64_137.node +0 -0
  222. package/node_modules/uWebSockets.js/uws_linux_arm64_147.node +0 -0
  223. package/node_modules/uWebSockets.js/uws_linux_x64_127.node +0 -0
  224. package/node_modules/uWebSockets.js/uws_linux_x64_137.node +0 -0
  225. package/node_modules/uWebSockets.js/uws_linux_x64_147.node +0 -0
  226. package/node_modules/uWebSockets.js/uws_win32_x64_127.node +0 -0
  227. package/node_modules/uWebSockets.js/uws_win32_x64_137.node +0 -0
  228. package/node_modules/uWebSockets.js/uws_win32_x64_147.node +0 -0
@@ -1,9 +1,9 @@
1
1
  import http from 'node:http'
2
2
  import { createRequire } from 'module'
3
3
  import { clearCache, init } from 'tardis-dev'
4
- import { App, DISABLED, TemplatedApp } from 'uWebSockets.js'
5
4
  import { healthCheck, replayHttp, replayNormalizedHttp } from './http/index.ts'
6
5
  import { replayNormalizedWS, replayWS, streamNormalizedWS } from './ws/index.ts'
6
+ import { MachineWebSocketServer } from './ws/server.ts'
7
7
  import { debug } from './debug.ts'
8
8
 
9
9
  const require = createRequire(import.meta.url)
@@ -11,7 +11,7 @@ const packageJson = require('../package.json') as { version: string }
11
11
 
12
12
  export class TardisMachine {
13
13
  private readonly _httpServer: http.Server
14
- private readonly _wsServer: TemplatedApp
14
+ private readonly _wsServer: MachineWebSocketServer
15
15
  private _eventLoopTimerId: NodeJS.Timeout | undefined = undefined
16
16
 
17
17
  constructor(private readonly options: Options) {
@@ -43,55 +43,33 @@ export class TardisMachine {
43
43
  // set timeout to 0 meaning infinite http timout - streaming may take some time expecially for longer date ranges
44
44
  this._httpServer.timeout = 0
45
45
 
46
- const wsRoutes = {
46
+ this._wsServer = new MachineWebSocketServer({
47
47
  '/ws-replay': replayWS,
48
48
  '/ws-replay-normalized': replayNormalizedWS,
49
49
  '/ws-stream-normalized': streamNormalizedWS
50
- } as any
51
-
52
- this._wsServer = App().ws('/*', {
53
- compression: DISABLED,
54
- maxPayloadLength: 512 * 1024,
55
- idleTimeout: 60,
56
- maxBackpressure: 5 * 1024 * 1024,
57
- closeOnBackpressureLimit: true,
58
- upgrade: (res: any, req: any, context: any) => {
59
- res.upgrade(
60
- { req },
61
- req.getHeader('sec-websocket-key'),
62
- req.getHeader('sec-websocket-protocol'),
63
- req.getHeader('sec-websocket-extensions'),
64
- context
65
- )
66
- },
67
- open: (ws: any) => {
68
- const path = ws.req.getUrl().toLocaleLowerCase()
69
- ws.closed = false
70
- const matchingRoute = wsRoutes[path]
71
-
72
- if (matchingRoute !== undefined) {
73
- matchingRoute(ws, ws.req)
74
- } else {
75
- ws.end(1008)
76
- }
77
- },
78
-
79
- message: (ws: any, message: ArrayBuffer) => {
80
- if (ws.onmessage !== undefined) {
81
- ws.onmessage(message)
82
- }
83
- },
84
-
85
- close: (ws: any) => {
86
- ws.closed = true
87
- if (ws.onclose !== undefined) {
88
- ws.onclose()
89
- }
90
- }
91
- } as any)
50
+ })
92
51
  }
93
52
 
94
53
  public async start(port: number) {
54
+ if (this.options.clearCache) {
55
+ await clearCache()
56
+ }
57
+
58
+ await new Promise<void>((resolve, reject) => {
59
+ this._httpServer.once('error', reject)
60
+ this._httpServer.listen(port, () => {
61
+ this._httpServer.removeListener('error', reject)
62
+ resolve()
63
+ })
64
+ })
65
+
66
+ try {
67
+ await this._wsServer.listen(port + 1)
68
+ } catch (error) {
69
+ await new Promise<void>((resolve) => this._httpServer.close(() => resolve()))
70
+ throw error
71
+ }
72
+
95
73
  let start = process.hrtime()
96
74
  const interval = 500
97
75
 
@@ -108,30 +86,11 @@ export class TardisMachine {
108
86
 
109
87
  start = process.hrtime()
110
88
  }, interval)
111
-
112
- if (this.options.clearCache) {
113
- await clearCache()
114
- }
115
-
116
- await new Promise<void>((resolve, reject) => {
117
- try {
118
- this._httpServer.on('error', reject)
119
- this._httpServer.listen(port, () => {
120
- this._wsServer.listen(port + 1, (listenSocket) => {
121
- if (listenSocket) {
122
- resolve()
123
- } else {
124
- reject(new Error('ws server did not start'))
125
- }
126
- })
127
- })
128
- } catch (e) {
129
- reject(e)
130
- }
131
- })
132
89
  }
133
90
 
134
91
  public async stop() {
92
+ await this._wsServer.close()
93
+
135
94
  await new Promise<void>((resolve, reject) => {
136
95
  this._httpServer.close((err) => {
137
96
  err ? reject(err) : resolve()
package/src/ws/replay.ts CHANGED
@@ -1,15 +1,15 @@
1
1
  import { decode } from 'node:querystring'
2
2
  import { combine, Exchange, replay, ReplayOptions } from 'tardis-dev'
3
- import type { HttpRequest } from 'uWebSockets.js'
4
3
  import { debug } from '../debug.ts'
5
4
  import { wait } from '../helpers.ts'
5
+ import type { MachineWebSocket } from './server.ts'
6
6
  import { SubscriptionMapper, subscriptionsMappers } from './subscriptionsmappers.ts'
7
7
 
8
8
  const replaySessions: { [sessionKey: string]: ReplaySession | undefined } = {}
9
9
  let sessionsCounter = 0
10
10
 
11
- export function replayWS(ws: any, req: HttpRequest) {
12
- const parsedQuery = decode(req.getQuery())
11
+ export function replayWS(ws: MachineWebSocket, query: string) {
12
+ const parsedQuery = decode(query)
13
13
  const from = parsedQuery['from'] as string
14
14
  const to = parsedQuery['to'] as string
15
15
  const exchange = parsedQuery['exchange'] as Exchange
@@ -94,10 +94,12 @@ class ReplaySession {
94
94
  })
95
95
 
96
96
  for await (const { message } of messages) {
97
+ if (connection.ws.closed) break
98
+
97
99
  const success = connection.ws.send(message)
98
100
  // handle backpressure in case of slow clients
99
101
  if (!success) {
100
- while (connection.ws.getBufferedAmount() > 0) {
102
+ while (!connection.ws.closed && connection.ws.getBufferedAmount() > 0) {
101
103
  await wait(1)
102
104
  }
103
105
  }
@@ -122,10 +124,13 @@ class ReplaySession {
122
124
  })
123
125
 
124
126
  for await (const { ws, message } of combine(...messagesWithConnections)) {
127
+ if (this._connections.every((connection) => connection.ws.closed)) break
128
+ if (ws.closed) continue
129
+
125
130
  const success = ws.send(message)
126
131
  // handle backpressure in case of slow clients
127
132
  if (!success) {
128
- while (ws.getBufferedAmount() > 0) {
133
+ while (!ws.closed && ws.getBufferedAmount() > 0) {
129
134
  await wait(1)
130
135
  }
131
136
  }
@@ -155,7 +160,7 @@ class ReplaySession {
155
160
  }
156
161
 
157
162
  // let's wait until buffer is empty before closing normal connections
158
- while (!error && connection.ws.getBufferedAmount() > 0) {
163
+ while (!error && !connection.ws.closed && connection.ws.getBufferedAmount() > 0) {
159
164
  await wait(100)
160
165
  }
161
166
 
@@ -174,7 +179,7 @@ class WebsocketConnection {
174
179
  public subscriptionsCount = 0
175
180
 
176
181
  constructor(
177
- public readonly ws: any,
182
+ public readonly ws: MachineWebSocket,
178
183
  exchange: Exchange,
179
184
  from: string,
180
185
  to: string
@@ -212,7 +217,7 @@ class WebsocketConnection {
212
217
  return `${JSON.stringify(this.replayOptions)}`
213
218
  }
214
219
 
215
- private _convertSubscribeRequestToFilter(messageRaw: ArrayBuffer) {
220
+ private _convertSubscribeRequestToFilter(messageRaw: Buffer) {
216
221
  const message = Buffer.from(messageRaw).toString()
217
222
  try {
218
223
  const messageDeserialized = JSON.parse(message)
@@ -1,14 +1,14 @@
1
1
  import { decode } from 'node:querystring'
2
2
  import { combine, compute, replayNormalized } from 'tardis-dev'
3
- import type { HttpRequest } from 'uWebSockets.js'
4
3
  import { debug } from '../debug.ts'
5
4
  import { constructDataTypeFilter, getComputables, getNormalizers, ReplayNormalizedRequestOptions, wait } from '../helpers.ts'
5
+ import type { MachineWebSocket } from './server.ts'
6
6
 
7
- export async function replayNormalizedWS(ws: any, req: HttpRequest) {
7
+ export async function replayNormalizedWS(ws: MachineWebSocket, query: string) {
8
8
  let messages: AsyncIterableIterator<any> | undefined
9
9
  try {
10
10
  const startTimestamp = new Date().getTime()
11
- const parsedQuery = decode(req.getQuery())
11
+ const parsedQuery = decode(query)
12
12
  const optionsString = parsedQuery['options'] as string
13
13
  const replayNormalizedOptions = JSON.parse(optionsString) as ReplayNormalizedRequestOptions
14
14
 
@@ -34,6 +34,8 @@ export async function replayNormalizedWS(ws: any, req: HttpRequest) {
34
34
  messages = messagesIterables.length === 1 ? messagesIterables[0] : combine(...messagesIterables)
35
35
 
36
36
  for await (const message of messages) {
37
+ if (ws.closed) return
38
+
37
39
  if (!filterByDataType(message)) {
38
40
  continue
39
41
  }
@@ -41,13 +43,13 @@ export async function replayNormalizedWS(ws: any, req: HttpRequest) {
41
43
  const success = ws.send(JSON.stringify(message))
42
44
  // handle backpressure in case of slow clients
43
45
  if (!success) {
44
- while (ws.getBufferedAmount() > 0) {
46
+ while (!ws.closed && ws.getBufferedAmount() > 0) {
45
47
  await wait(1)
46
48
  }
47
49
  }
48
50
  }
49
51
 
50
- while (ws.getBufferedAmount() > 0) {
52
+ while (!ws.closed && ws.getBufferedAmount() > 0) {
51
53
  await wait(100)
52
54
  }
53
55
 
@@ -63,7 +65,7 @@ export async function replayNormalizedWS(ws: any, req: HttpRequest) {
63
65
  } catch (e: any) {
64
66
  // this will underlying open WS connections
65
67
  if (messages !== undefined) {
66
- messages!.return!()
68
+ await messages.return?.()
67
69
  }
68
70
  if (!ws.closed) {
69
71
  ws.end(1011, e.toString())
@@ -0,0 +1,173 @@
1
+ import http from 'node:http'
2
+ import WebSocket, { WebSocketServer } from 'ws'
3
+
4
+ const IDLE_TIMEOUT_MS = 60_000
5
+ const HEARTBEAT_INTERVAL_MS = IDLE_TIMEOUT_MS / 2
6
+ const MAX_HEADER_BYTES = 20_000
7
+ const MAX_PAYLOAD_BYTES = 512 * 1024
8
+ const MAX_BACKPRESSURE_BYTES = 5 * 1024 * 1024
9
+ const MAX_CLOSE_REASON_BYTES = 123
10
+
11
+ export type MachineWebSocket = {
12
+ closed: boolean
13
+ onmessage?: (message: Buffer) => void
14
+ send(message: string | ArrayBuffer | Buffer): boolean
15
+ getBufferedAmount(): number
16
+ end(code?: number, reason?: string): void
17
+ }
18
+
19
+ export type MachineWebSocketRoute = (socket: MachineWebSocket, query: string) => void | Promise<void>
20
+
21
+ export class MachineWebSocketServer {
22
+ private readonly httpServer = http.createServer({ maxHeaderSize: MAX_HEADER_BYTES }, (_, response) => {
23
+ response.writeHead(426).end()
24
+ })
25
+ private readonly server = new WebSocketServer({
26
+ server: this.httpServer,
27
+ perMessageDeflate: false,
28
+ maxPayload: MAX_PAYLOAD_BYTES
29
+ })
30
+ private readonly connections = new Set<MachineWebSocketConnection>()
31
+ private heartbeatTimer: NodeJS.Timeout | undefined
32
+
33
+ constructor(private readonly routes: Record<string, MachineWebSocketRoute>) {
34
+ // `ws` re-emits errors from the supplied HTTP server; `listen` reports the same error to its caller.
35
+ this.server.on('error', () => {})
36
+ this.server.on('connection', (webSocket, request) => this.connect(webSocket, request.url))
37
+ }
38
+
39
+ public async listen(port: number) {
40
+ await new Promise<void>((resolve, reject) => {
41
+ this.httpServer.once('error', reject)
42
+ this.httpServer.listen(port, () => {
43
+ this.httpServer.removeListener('error', reject)
44
+ resolve()
45
+ })
46
+ })
47
+
48
+ this.heartbeatTimer = setInterval(() => {
49
+ for (const connection of this.connections) connection.checkHeartbeat()
50
+ }, HEARTBEAT_INTERVAL_MS)
51
+ }
52
+
53
+ public async close() {
54
+ if (this.heartbeatTimer !== undefined) {
55
+ clearInterval(this.heartbeatTimer)
56
+ this.heartbeatTimer = undefined
57
+ }
58
+ for (const connection of this.connections) connection.terminate()
59
+
60
+ await Promise.all([
61
+ new Promise<void>((resolve, reject) => {
62
+ this.server.close((error) => (error === undefined ? resolve() : reject(error)))
63
+ }),
64
+ new Promise<void>((resolve, reject) => {
65
+ this.httpServer.close((error) => (error === undefined ? resolve() : reject(error)))
66
+ })
67
+ ])
68
+ }
69
+
70
+ private connect(webSocket: WebSocket, requestUrl: string | undefined) {
71
+ const connection = new MachineWebSocketConnection(webSocket, () => this.connections.delete(connection))
72
+ this.connections.add(connection)
73
+
74
+ let url: URL
75
+ try {
76
+ url = new URL(requestUrl ?? '/', 'http://localhost')
77
+ } catch {
78
+ connection.end(1008)
79
+ return
80
+ }
81
+
82
+ const route = this.routes[url.pathname.toLowerCase()]
83
+ if (route === undefined) {
84
+ connection.end(1008)
85
+ return
86
+ }
87
+
88
+ try {
89
+ const result = route(connection, url.search.slice(1))
90
+ if (result !== undefined) void result.catch((error) => connection.end(1011, String(error)))
91
+ } catch (error) {
92
+ connection.end(1011, String(error))
93
+ }
94
+ }
95
+ }
96
+
97
+ class MachineWebSocketConnection implements MachineWebSocket {
98
+ public closed = false
99
+ public onmessage: ((message: Buffer) => void) | undefined
100
+ private receivedHeartbeat = true
101
+
102
+ constructor(
103
+ private readonly socket: WebSocket,
104
+ private readonly remove: () => void
105
+ ) {
106
+ socket.on('message', (message) => {
107
+ this.receivedHeartbeat = true
108
+ this.onmessage?.(Buffer.isBuffer(message) ? message : Buffer.from(message as ArrayBuffer))
109
+ })
110
+ socket.on('pong', () => (this.receivedHeartbeat = true))
111
+ // `ws` closes the connection after protocol and transport errors; cleanup happens in `close`.
112
+ socket.on('error', () => {})
113
+ socket.on('close', () => {
114
+ this.closed = true
115
+ this.remove()
116
+ })
117
+ }
118
+
119
+ public send(message: string | ArrayBuffer | Buffer) {
120
+ if (this.socket.readyState !== WebSocket.OPEN) return false
121
+
122
+ try {
123
+ // uWebSockets.js sent replay payload buffers as text frames by default.
124
+ this.socket.send(message, { binary: false, compress: false })
125
+ } catch {
126
+ this.terminate()
127
+ return false
128
+ }
129
+
130
+ if (this.socket.bufferedAmount > MAX_BACKPRESSURE_BYTES) {
131
+ this.end(1008, 'Too much backpressure')
132
+ return false
133
+ }
134
+
135
+ return this.socket.bufferedAmount === 0
136
+ }
137
+
138
+ public getBufferedAmount() {
139
+ return this.socket.bufferedAmount
140
+ }
141
+
142
+ public end(code = 1000, reason = '') {
143
+ if (this.socket.readyState !== WebSocket.OPEN) return
144
+
145
+ try {
146
+ this.socket.close(code, truncateCloseReason(reason))
147
+ } catch {
148
+ this.terminate()
149
+ }
150
+ }
151
+
152
+ public checkHeartbeat() {
153
+ if (!this.receivedHeartbeat) {
154
+ this.terminate()
155
+ return
156
+ }
157
+
158
+ this.receivedHeartbeat = false
159
+ if (this.socket.readyState === WebSocket.OPEN) this.socket.ping()
160
+ }
161
+
162
+ public terminate() {
163
+ if (this.socket.readyState !== WebSocket.CLOSED) this.socket.terminate()
164
+ }
165
+ }
166
+
167
+ function truncateCloseReason(reason: string) {
168
+ const encodedReason = Buffer.from(reason)
169
+ if (encodedReason.length <= MAX_CLOSE_REASON_BYTES) return reason
170
+
171
+ // Leave room for the UTF-8 replacement character if truncation splits a code point.
172
+ return encodedReason.subarray(0, MAX_CLOSE_REASON_BYTES - 3).toString()
173
+ }
@@ -1,15 +1,15 @@
1
1
  import { decode } from 'node:querystring'
2
2
  import { combine, compute, Exchange, streamNormalized } from 'tardis-dev'
3
- import type { HttpRequest } from 'uWebSockets.js'
4
3
  import { debug } from '../debug.ts'
5
4
  import { constructDataTypeFilter, getComputables, getNormalizers, StreamNormalizedRequestOptions, wait } from '../helpers.ts'
5
+ import type { MachineWebSocket } from './server.ts'
6
6
 
7
- export async function streamNormalizedWS(ws: any, req: HttpRequest) {
7
+ export async function streamNormalizedWS(ws: MachineWebSocket, query: string) {
8
8
  let messages: AsyncIterableIterator<any> | undefined
9
9
 
10
10
  try {
11
11
  const startTimestamp = new Date().getTime()
12
- const parsedQuery = decode(req.getQuery())
12
+ const parsedQuery = decode(query)
13
13
  const optionsString = parsedQuery['options'] as string
14
14
  const streamNormalizedOptions = JSON.parse(optionsString) as StreamNormalizedRequestOptions
15
15
 
@@ -84,7 +84,7 @@ export async function streamNormalizedWS(ws: any, req: HttpRequest) {
84
84
  retries = 0
85
85
  bufferedAmount = 0
86
86
  // handle backpressure in case of slow clients
87
- while ((bufferedAmount = ws.getBufferedAmount()) > 0) {
87
+ while (!ws.closed && (bufferedAmount = ws.getBufferedAmount()) > 0) {
88
88
  retries += 1
89
89
  const isState = new Date().valueOf() - message.localTimestamp.valueOf() >= 6
90
90
 
@@ -101,6 +101,8 @@ export async function streamNormalizedWS(ws: any, req: HttpRequest) {
101
101
  await wait(3 * retries)
102
102
  }
103
103
 
104
+ if (ws.closed) return
105
+
104
106
  ws.send(JSON.stringify(message))
105
107
 
106
108
  if (message.type !== 'disconnect') {
@@ -108,7 +110,7 @@ export async function streamNormalizedWS(ws: any, req: HttpRequest) {
108
110
  }
109
111
  }
110
112
 
111
- while (ws.getBufferedAmount() > 0) {
113
+ while (!ws.closed && ws.getBufferedAmount() > 0) {
112
114
  await wait(100)
113
115
  }
114
116
 
@@ -131,7 +133,7 @@ export async function streamNormalizedWS(ws: any, req: HttpRequest) {
131
133
  } finally {
132
134
  // this will close underlying open WS connections
133
135
  if (messages !== undefined) {
134
- messages!.return!()
136
+ await messages.return?.()
135
137
  }
136
138
  }
137
139
  }
@@ -1,4 +1,4 @@
1
- import { Exchange, Filter } from 'tardis-dev'
1
+ import type { Exchange, Filter } from 'tardis-dev'
2
2
 
3
3
  // https://www.bitmex.com/app/wsAPI
4
4
  const bitmexMapper: SubscriptionMapper = {
@@ -1,18 +0,0 @@
1
- Copyright 2014–present Olivier Lalonde <olalonde@gmail.com>, James Talmage <james@talmage.io>, Ruben Verborgh
2
-
3
- Permission is hereby granted, free of charge, to any person obtaining a copy of
4
- this software and associated documentation files (the "Software"), to deal in
5
- the Software without restriction, including without limitation the rights to
6
- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
7
- of the Software, and to permit persons to whom the Software is furnished to do
8
- so, subject to the following conditions:
9
-
10
- The above copyright notice and this permission notice shall be included in all
11
- copies or substantial portions of the Software.
12
-
13
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
17
- WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
18
- IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -1,157 +0,0 @@
1
- ## Follow Redirects
2
-
3
- Drop-in replacement for Node's `http` and `https` modules that automatically follows redirects.
4
-
5
- [![npm version](https://img.shields.io/npm/v/follow-redirects.svg)](https://www.npmjs.com/package/follow-redirects)
6
- [![Build Status](https://github.com/follow-redirects/follow-redirects/workflows/CI/badge.svg)](https://github.com/follow-redirects/follow-redirects/actions)
7
- [![Coverage Status](https://coveralls.io/repos/follow-redirects/follow-redirects/badge.svg?branch=master)](https://coveralls.io/r/follow-redirects/follow-redirects?branch=master)
8
- [![npm downloads](https://img.shields.io/npm/dm/follow-redirects.svg)](https://www.npmjs.com/package/follow-redirects)
9
- [![Sponsor on GitHub](https://img.shields.io/static/v1?label=Sponsor&message=%F0%9F%92%96&logo=GitHub)](https://github.com/sponsors/RubenVerborgh)
10
-
11
- `follow-redirects` provides [request](https://nodejs.org/api/http.html#http_http_request_options_callback) and [get](https://nodejs.org/api/http.html#http_http_get_options_callback)
12
- methods that behave identically to those found on the native [http](https://nodejs.org/api/http.html#http_http_request_options_callback) and [https](https://nodejs.org/api/https.html#https_https_request_options_callback)
13
- modules, with the exception that they will seamlessly follow redirects.
14
-
15
- ```javascript
16
- const { http, https } = require('follow-redirects');
17
-
18
- http.get('http://en.wikipedia.org/', response => {
19
- response.on('data', chunk => {
20
- console.log(chunk);
21
- });
22
- }).on('error', err => {
23
- console.error(err);
24
- });
25
- ```
26
-
27
- You can inspect the final redirected URL through the `responseUrl` property on the `response`.
28
- If no redirection happened, `responseUrl` is the original request URL.
29
-
30
- ```javascript
31
- const request = https.request({
32
- host: 'en.wikipedia.org',
33
- path: '/',
34
- }, response => {
35
- console.log(response.responseUrl);
36
- // 'http://duckduckgo.com/robots.txt'
37
- });
38
- request.end();
39
- ```
40
-
41
- ## Options
42
- ### Global options
43
- Global options are set directly on the `follow-redirects` module:
44
-
45
- ```javascript
46
- const followRedirects = require('follow-redirects');
47
- followRedirects.maxRedirects = 10;
48
- followRedirects.maxBodyLength = 20 * 1024 * 1024; // 20 MB
49
- ```
50
-
51
- The following global options are supported:
52
-
53
- - `maxRedirects` (default: `21`) – sets the maximum number of allowed redirects; if exceeded, an error will be emitted.
54
-
55
- - `maxBodyLength` (default: 10MB) – sets the maximum size of the request body; if exceeded, an error will be emitted.
56
-
57
- ### Per-request options
58
- Per-request options are set by passing an `options` object:
59
-
60
- ```javascript
61
- const url = require('url');
62
- const { http, https } = require('follow-redirects');
63
-
64
- const options = url.parse('http://en.wikipedia.org/');
65
- options.maxRedirects = 10;
66
- options.beforeRedirect = (options, response, request) => {
67
- // Use this to adjust the request options upon redirecting,
68
- // to inspect the latest response headers,
69
- // or to cancel the request by throwing an error
70
-
71
- // response.headers = the redirect response headers
72
- // response.statusCode = the redirect response code (eg. 301, 307, etc.)
73
-
74
- // request.url = the requested URL that resulted in a redirect
75
- // request.headers = the headers in the request that resulted in a redirect
76
- // request.method = the method of the request that resulted in a redirect
77
- if (options.hostname === "example.org") {
78
- options.auth = "user:password";
79
- }
80
- };
81
- http.request(options);
82
- ```
83
-
84
- In addition to the [standard HTTP](https://nodejs.org/api/http.html#http_http_request_options_callback) and [HTTPS options](https://nodejs.org/api/https.html#https_https_request_options_callback),
85
- the following per-request options are supported:
86
- - `followRedirects` (default: `true`) – whether redirects should be followed.
87
-
88
- - `maxRedirects` (default: `21`) – sets the maximum number of allowed redirects; if exceeded, an error will be emitted.
89
-
90
- - `maxBodyLength` (default: 10MB) – sets the maximum size of the request body; if exceeded, an error will be emitted.
91
-
92
- - `beforeRedirect` (default: `undefined`) – optionally change the request `options` on redirects, or abort the request by throwing an error.
93
-
94
- - `agents` (default: `undefined`) – sets the `agent` option per protocol, since HTTP and HTTPS use different agents. Example value: `{ http: new http.Agent(), https: new https.Agent() }`
95
-
96
- - `trackRedirects` (default: `false`) – whether to store the redirected response details into the `redirects` array on the response object.
97
-
98
- - `sensitiveHeaders` (default: `[]`) – names of headers to omit when making redirected requests (such as `X-API-Key`, `X-Auth-Token`…)
99
-
100
-
101
- ### Advanced usage
102
- By default, `follow-redirects` will use the Node.js default implementations
103
- of [`http`](https://nodejs.org/api/http.html)
104
- and [`https`](https://nodejs.org/api/https.html).
105
- To enable features such as caching and/or intermediate request tracking,
106
- you might instead want to wrap `follow-redirects` around custom protocol implementations:
107
-
108
- ```javascript
109
- const { http, https } = require('follow-redirects').wrap({
110
- http: require('your-custom-http'),
111
- https: require('your-custom-https'),
112
- });
113
- ```
114
-
115
- Such custom protocols only need an implementation of the `request` method.
116
-
117
- ## Browser Usage
118
-
119
- Due to the way the browser works,
120
- the `http` and `https` browser equivalents perform redirects by default.
121
-
122
- By requiring `follow-redirects` this way:
123
- ```javascript
124
- const http = require('follow-redirects/http');
125
- const https = require('follow-redirects/https');
126
- ```
127
- you can easily tell webpack and friends to replace
128
- `follow-redirect` by the built-in versions:
129
-
130
- ```json
131
- {
132
- "follow-redirects/http" : "http",
133
- "follow-redirects/https" : "https"
134
- }
135
- ```
136
-
137
- ## Contributing
138
-
139
- Pull Requests are always welcome. Please [file an issue](https://github.com/follow-redirects/follow-redirects/issues)
140
- detailing your proposal before you invest your valuable time. Additional features and bug fixes should be accompanied
141
- by tests. You can run the test suite locally with a simple `npm test` command.
142
-
143
- ## Debug Logging
144
-
145
- `follow-redirects` uses the excellent [debug](https://www.npmjs.com/package/debug) for logging. To turn on logging
146
- set the environment variable `DEBUG=follow-redirects` for debug output from just this module. When running the test
147
- suite it is sometimes advantageous to set `DEBUG=*` to see output from the express server as well.
148
-
149
- ## Authors
150
-
151
- - [Ruben Verborgh](https://ruben.verborgh.org/)
152
- - [Olivier Lalonde](mailto:olalonde@gmail.com)
153
- - [James Talmage](mailto:james@talmage.io)
154
-
155
- ## License
156
-
157
- [MIT License](https://github.com/follow-redirects/follow-redirects/blob/master/LICENSE)
@@ -1,15 +0,0 @@
1
- var debug;
2
-
3
- module.exports = function () {
4
- if (!debug) {
5
- try {
6
- /* eslint global-require: off */
7
- debug = require("debug")("follow-redirects");
8
- }
9
- catch (error) { /* */ }
10
- if (typeof debug !== "function") {
11
- debug = function () { /* */ };
12
- }
13
- }
14
- debug.apply(null, arguments);
15
- };
@@ -1 +0,0 @@
1
- module.exports = require("./").http;
@@ -1 +0,0 @@
1
- module.exports = require("./").https;