vortexpy 4.0.23__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. vortex/DataWrapTuple.py +27 -0
  2. vortex/DeferUtil.py +224 -0
  3. vortex/Jsonable.py +304 -0
  4. vortex/Payload.py +115 -0
  5. vortex/PayloadEndpoint.py +216 -0
  6. vortex/PayloadEndpointTest.py +164 -0
  7. vortex/PayloadEnvelope.py +166 -0
  8. vortex/PayloadFilterKeys.py +15 -0
  9. vortex/PayloadIO.py +152 -0
  10. vortex/PayloadPriority.py +11 -0
  11. vortex/PayloadResponse.py +173 -0
  12. vortex/PayloadResponseTest.py +39 -0
  13. vortex/PayloadTest.py +136 -0
  14. vortex/PerformTestActionTuple.py +30 -0
  15. vortex/SerialiseUtil.py +273 -0
  16. vortex/TestTuple.py +61 -0
  17. vortex/Tuple.py +968 -0
  18. vortex/TupleAction.py +72 -0
  19. vortex/TupleActionVortex.py +5 -0
  20. vortex/TupleDataForTest.py +25 -0
  21. vortex/TuplePerfTest.py +71 -0
  22. vortex/TuplePolymorphicTupleTypeFieldArgTest.py +74 -0
  23. vortex/TupleRestfulTest.py +140 -0
  24. vortex/TupleSelector.py +105 -0
  25. vortex/TupleTest.py +127 -0
  26. vortex/VortexABC.py +55 -0
  27. vortex/VortexClientHttp.py +308 -0
  28. vortex/VortexClientHttpTest.py +71 -0
  29. vortex/VortexClientTcp.py +355 -0
  30. vortex/VortexClientWebsocketFactory.py +383 -0
  31. vortex/VortexConnectionABC.py +88 -0
  32. vortex/VortexFactory.py +932 -0
  33. vortex/VortexPayloadProtocol.py +180 -0
  34. vortex/VortexServer.py +367 -0
  35. vortex/VortexServerConnection.py +127 -0
  36. vortex/VortexServerHttpResource.py +193 -0
  37. vortex/VortexServerTcp.py +104 -0
  38. vortex/VortexServerWebsocket.py +287 -0
  39. vortex/VortexTcpMemoryLeakPayloadTest.py +611 -0
  40. vortex/VortexTcpMemoryLeakRawTest.py +87 -0
  41. vortex/VortexTest.py +22 -0
  42. vortex/VortexUtil.py +314 -0
  43. vortex/VortexWritePushProducer.py +321 -0
  44. vortex/__init__.py +10 -0
  45. vortex/data_loader/TupleDataLiveLockManager.py +383 -0
  46. vortex/data_loader/TupleDataLoader.py +326 -0
  47. vortex/data_loader/TupleDataLoaderDelegate.py +90 -0
  48. vortex/data_loader/TupleDataLoaderTupleABC.py +10 -0
  49. vortex/data_loader/TupleDataLoaderTuples.py +53 -0
  50. vortex/data_loader/__init__.py +0 -0
  51. vortex/event_bus/TupleChangeEventABC.py +15 -0
  52. vortex/event_bus/TupleChangeEventBus.py +192 -0
  53. vortex/event_bus/TupleChangeEventBusObserverABC.py +11 -0
  54. vortex/event_bus/__init__.py +0 -0
  55. vortex/handler/TupleActionProcessor.py +270 -0
  56. vortex/handler/TupleActionProcessorProxy.py +218 -0
  57. vortex/handler/TupleActionProcessorTestHandler.py +27 -0
  58. vortex/handler/TupleDataActionClient.py +67 -0
  59. vortex/handler/TupleDataObservableCache.py +201 -0
  60. vortex/handler/TupleDataObservableHandler.py +371 -0
  61. vortex/handler/TupleDataObservableProxyHandler.py +340 -0
  62. vortex/handler/TupleDataObservableTestHandler.py +60 -0
  63. vortex/handler/TupleDataObserverClient.py +158 -0
  64. vortex/handler/VortexJSTupleLoaderTestHandler.py +42 -0
  65. vortex/handler/__init__.py +0 -0
  66. vortex/restful/GzippedDataHttpClient.py +170 -0
  67. vortex/restful/GzippedDataHttpClientTest.py +103 -0
  68. vortex/restful/GzippedPayloadHttpClient.py +181 -0
  69. vortex/restful/RestfulHttpClient.py +107 -0
  70. vortex/restful/RestfulHttpClientTest.py +60 -0
  71. vortex/restful/RestfulResource.py +153 -0
  72. vortex/restful/RestfulResourceTest.py +100 -0
  73. vortex/restful/__init__.py +0 -0
  74. vortex/rpc/RPC.py +446 -0
  75. vortex/rpc/RPCTest.py +12 -0
  76. vortex/rpc/__init__.py +0 -0
  77. vortex/sqla_orm/OrmCrudHandler.py +431 -0
  78. vortex/sqla_orm/TupleProviderForDb.py +42 -0
  79. vortex/sqla_orm/VortexTupleJson.py +24 -0
  80. vortex/sqla_orm/__init__.py +0 -0
  81. vortex/storage/TupleStorageSqlite.py +359 -0
  82. vortex/storage/__init__.py +0 -0
  83. vortexpy-4.0.23.dist-info/LICENSE +21 -0
  84. vortexpy-4.0.23.dist-info/METADATA +43 -0
  85. vortexpy-4.0.23.dist-info/RECORD +87 -0
  86. vortexpy-4.0.23.dist-info/WHEEL +5 -0
  87. vortexpy-4.0.23.dist-info/top_level.txt +1 -0
@@ -0,0 +1,932 @@
1
+ import logging
2
+ from collections import defaultdict
3
+ from collections import namedtuple
4
+ from datetime import datetime
5
+ from typing import Dict
6
+ from typing import List
7
+ from typing import Optional
8
+ from typing import Union
9
+
10
+ import pytz
11
+ from OpenSSL import SSL
12
+ from reactivex.subject import Subject
13
+ from twisted.internet import reactor
14
+ from twisted.internet import ssl
15
+ from twisted.internet._sslverify import trustRootFromCertificates
16
+ from twisted.internet.defer import Deferred
17
+ from twisted.internet.defer import DeferredList
18
+ from twisted.internet.defer import inlineCallbacks
19
+ from twisted.internet.defer import succeed
20
+ from twisted.internet.task import deferLater
21
+ from twisted.python.failure import Failure
22
+ from txhttputil.util.PemUtil import parsePemBundleForClient
23
+ from txhttputil.util.PemUtil import parsePemBundleForServer
24
+ from txhttputil.util.PemUtil import parsePemBundleForTrustedPeers
25
+ from txhttputil.util.SslUtil import buildCertificateOptionsForTwisted
26
+ from txhttputil.util.SslUtil import parseTrustRootFromBundle
27
+ from txwebsocket.txws import WebSocketFactory
28
+
29
+ from vortex.DeferUtil import yesMainThread
30
+ from vortex.PayloadEnvelope import PayloadEnvelope
31
+ from vortex.PayloadEnvelope import VortexMsgList
32
+ from vortex.PayloadIO import PayloadIO
33
+ from vortex.PayloadPriority import DEFAULT_PRIORITY
34
+ from vortex.VortexABC import VortexABC
35
+ from vortex.VortexABC import VortexInfo
36
+ from vortex.VortexClientHttp import VortexClientHttp
37
+ from vortex.VortexClientTcp import VortexClientTcp
38
+ from vortex.VortexClientWebsocketFactory import VortexClientWebsocketFactory
39
+ from vortex.VortexServer import VortexServer
40
+ from vortex.VortexServerHttpResource import VortexServerHttpResource
41
+ from vortex.VortexServerTcp import VortexTcpServerFactory
42
+ from vortex.VortexServerWebsocket import VortexWebSocketUpgradeResource
43
+ from vortex.VortexServerWebsocket import VortexWebsocketServerFactory
44
+ from vortex.VortexServerWebsocket import VortexWrappedWebSocketFactory
45
+
46
+ logger = logging.getLogger(__name__)
47
+
48
+ broadcast = None
49
+
50
+ browserVortexName = "browser"
51
+
52
+
53
+ class NoVortexException(Exception):
54
+ """No Vortex Exception
55
+
56
+ This is raised when a remote vortex doesn't exist.
57
+
58
+ """
59
+
60
+
61
+ _ConnectionDateSuccess = namedtuple(
62
+ "_ConnectionDateSuccess", ("dateTime", "success", "ip")
63
+ )
64
+
65
+
66
+ class _VortexConnectionRateLimit:
67
+ def __init__(self):
68
+ self._connections: list[_ConnectionDateSuccess] = []
69
+ self._peerConnectionLimitPerIp = None
70
+ self._newConnectionsPerWindow = 5
71
+ self._rollingNewConnectionWindowSeconds = 10.0
72
+
73
+ def _hasHitPeerConnectionLimit(
74
+ self, connectionCountForPeer: int, fromIp: str
75
+ ) -> bool:
76
+ if not self._peerConnectionLimitPerIp:
77
+ return True
78
+
79
+ if connectionCountForPeer + 1 <= self._peerConnectionLimitPerIp:
80
+ return True
81
+
82
+ logger.info(
83
+ "Closing new connection from %s,"
84
+ " this peer has %s connections, our limit is %s",
85
+ fromIp,
86
+ connectionCountForPeer,
87
+ self._peerConnectionLimitPerIp,
88
+ )
89
+
90
+ return False
91
+
92
+ def canConnect(
93
+ self,
94
+ connectionCountForPeer: int,
95
+ totalConnectionCount: int,
96
+ fromIp: str,
97
+ ) -> bool:
98
+ if not self._hasHitPeerConnectionLimit(connectionCountForPeer, fromIp):
99
+ return False
100
+
101
+ now = datetime.now(pytz.utc)
102
+ self._connections = list(
103
+ filter(
104
+ lambda item: (now - item.dateTime).total_seconds()
105
+ < self._rollingNewConnectionWindowSeconds,
106
+ self._connections,
107
+ )
108
+ )
109
+
110
+ connectedInWindow = len(
111
+ list(filter(lambda item: item.success, self._connections))
112
+ )
113
+
114
+ if connectedInWindow < self._newConnectionsPerWindow:
115
+ self._connections.append(_ConnectionDateSuccess(now, True, fromIp))
116
+ return True
117
+
118
+ self._connections.append(_ConnectionDateSuccess(now, False, fromIp))
119
+
120
+ earliestDate = min([c.dateTime for c in self._connections])
121
+ logger.info(
122
+ "Closing new connection from %s,"
123
+ " we've allowed %s"
124
+ " and closed %s connections in the last %s seconds,"
125
+ " we have %s connections",
126
+ fromIp,
127
+ connectedInWindow,
128
+ len(self._connections) - connectedInWindow,
129
+ now - earliestDate,
130
+ totalConnectionCount,
131
+ )
132
+
133
+ return False
134
+
135
+ def setNewConnectionsPerWindow(self, limit):
136
+ self._newConnectionsPerWindow = limit
137
+
138
+ def setPeerConnectionLimitPerIp(self, limit):
139
+ self._peerConnectionLimitPerIp = limit
140
+
141
+ def setRollingNewConnectionWindowSeconds(self, seconds):
142
+ self._rollingNewConnectionWindowSeconds = seconds
143
+
144
+
145
+ VortexUuidList = List[str]
146
+
147
+
148
+ class _VortexFactoryVortexInfo:
149
+ def __init__(self):
150
+ self.__vortexServersByName: Dict[str, List[VortexServer]] = defaultdict(
151
+ list
152
+ )
153
+ self.__vortexClientsByName: Dict[str, List[VortexABC]] = defaultdict(
154
+ list
155
+ )
156
+
157
+ self.__vortexServers: tuple[VortexServer] = tuple()
158
+ self.__vortexClients: tuple[VortexABC] = tuple()
159
+ self.__allVortexes: tuple[VortexABC] = tuple()
160
+
161
+ self.__remoteVortexUuidsByName = defaultdict(set)
162
+ self.__remoteVortexNameByUuid = {}
163
+
164
+ self.__remoteVortexUuids: set[str] = set()
165
+ self.__inboundConnectionCount: int = 0
166
+ self.__inboundConnectionCountPerIp = defaultdict(lambda: 0)
167
+
168
+ self.__localVortexUuidsByName = defaultdict(set)
169
+ self.__localVortexNameByUuid = {}
170
+
171
+ def addVortexServer(self, vortexName: str, vortex: VortexServer):
172
+ vortexes = self.__vortexServersByName[vortexName]
173
+
174
+ if vortex in vortexes:
175
+ raise Exception("Vortex is already registered, %s", vortex)
176
+
177
+ vortexes.append(vortex)
178
+ self.rebuildStructs()
179
+
180
+ def addVortexClient(self, vortexName: str, vortex: VortexABC):
181
+ vortexes = self.__vortexClientsByName[vortexName]
182
+
183
+ if vortex in vortexes:
184
+ raise Exception("Vortex is already registered, %s", vortex)
185
+
186
+ vortexes.append(vortex)
187
+ self.rebuildStructs()
188
+
189
+ def rebuildStructs(self):
190
+ vortexServers = []
191
+ for vortexes in self.__vortexServersByName.values():
192
+ vortexServers.extend(vortexes)
193
+
194
+ self.__vortexServers = tuple(vortexServers)
195
+
196
+ vortexClients = []
197
+ for vortexes in self.__vortexClientsByName.values():
198
+ vortexClients.extend(vortexes)
199
+
200
+ self.__vortexClients = tuple(vortexClients)
201
+
202
+ self.__allVortexes = self.__vortexServers + self.__vortexClients
203
+
204
+ self.__remoteVortexUuidsByName: dict[
205
+ str, set[tuple[VortexABC, str]]
206
+ ] = defaultdict(set)
207
+
208
+ self.__remoteVortexNameByUuid: dict[str, tuple[VortexABC, str]] = {}
209
+
210
+ self.__localVortexUuidsByName: dict[str, set[tuple[VortexABC, str]]] = (
211
+ defaultdict(set)
212
+ )
213
+
214
+ self.__localVortexNameByUuid: dict[str, tuple[VortexABC, str]] = {}
215
+
216
+ self.__inboundConnectionCount: int = 0
217
+ self.__inboundConnectionCountPerIp = defaultdict(lambda: 0)
218
+ for vortex in self.__vortexServers:
219
+ self.__inboundConnectionCount += len(vortex.connections)
220
+ for conn in vortex.connections.values():
221
+ self.__inboundConnectionCountPerIp[conn.ip] += 1
222
+
223
+ for vortex in self.allVortexes:
224
+ for remoteVortexInfo in vortex.remoteVortexInfo:
225
+ self.__remoteVortexUuidsByName[remoteVortexInfo.name].add(
226
+ (vortex, remoteVortexInfo.uuid)
227
+ )
228
+ assert (
229
+ remoteVortexInfo.uuid not in self.__remoteVortexNameByUuid
230
+ ), "Remote UUID is already registered - all vortexes %s" % str(
231
+ self.allVortexes
232
+ )
233
+ self.__remoteVortexNameByUuid[remoteVortexInfo.uuid] = (
234
+ vortex,
235
+ remoteVortexInfo.name,
236
+ )
237
+
238
+ self.__localVortexUuidsByName[vortex.localVortexInfo.name].add(
239
+ (vortex, vortex.localVortexInfo.uuid)
240
+ )
241
+ assert (
242
+ vortex.localVortexInfo.uuid not in self.__localVortexNameByUuid
243
+ ), "Local UUID is already registered - all vortexes %s" % str(
244
+ self.allVortexes
245
+ )
246
+ self.__localVortexNameByUuid[vortex.localVortexInfo.uuid] = (
247
+ vortex,
248
+ vortex.localVortexInfo.name,
249
+ )
250
+
251
+ self.__remoteVortexUuids = set(self.__remoteVortexNameByUuid)
252
+
253
+ def getRemoteVortexInfos(
254
+ self, name=None, uuid=None
255
+ ) -> list[tuple[VortexABC, str]]:
256
+ assert name or uuid, "We expect a name or uuid"
257
+
258
+ # If we're passed a UUID, then there will only be one result
259
+ if uuid:
260
+ vortex, foundName = self.__remoteVortexNameByUuid.get(
261
+ uuid, (None, None)
262
+ )
263
+ if not foundName:
264
+ return []
265
+ return [(vortex, uuid)]
266
+
267
+ # Name is asserted
268
+ return [
269
+ (vortex, foundUuid)
270
+ for vortex, foundUuid in self.__remoteVortexUuidsByName.get(
271
+ name, []
272
+ )
273
+ ]
274
+
275
+ def getRemoteVortexIpByUuid(self, vortexUuid: str, vortexName=None) -> str:
276
+ from vortex.VortexConnectionABC import VortexConnectionABC
277
+
278
+ targetConns: list[VortexConnectionABC] = []
279
+ for server in self.vortexSevers:
280
+ for conn in server.connections.values():
281
+ if (
282
+ not conn.closed
283
+ and not conn.timedOut
284
+ and conn.remoteVortexUuid == vortexUuid
285
+ and (
286
+ conn.remoteVortexName == vortexName
287
+ or vortexName is None
288
+ )
289
+ ):
290
+ targetConns.append(conn)
291
+
292
+ if len(targetConns) > 1:
293
+ logger.warning(
294
+ "Multiple vortexes found for uuid=%s, name=%s,"
295
+ " choosing latest connection",
296
+ vortexUuid,
297
+ vortexName,
298
+ )
299
+ targetConns.sort(key=lambda c: c.connectDateTime)
300
+ # As of Python3.6, dict are ordered, so the newest is the last.
301
+ return targetConns[-1].ip
302
+
303
+ if not targetConns:
304
+ raise NoVortexException(
305
+ f"Can not find vortex with UUID {vortexUuid}"
306
+ )
307
+
308
+ return targetConns[0].ip
309
+
310
+ def getRemoteVortexUuidByIp(self, ip: str, vortexName=None) -> str:
311
+ # Ignore the port if any
312
+ ip = ip.split(":")[0]
313
+
314
+ from vortex.VortexConnectionABC import VortexConnectionABC
315
+
316
+ targetConns: list[VortexConnectionABC] = []
317
+ for server in self.vortexSevers:
318
+ for conn in server.connections.values():
319
+ if (
320
+ not conn.closed
321
+ and not conn.timedOut
322
+ and conn.ip == ip
323
+ and (
324
+ conn.remoteVortexName == vortexName
325
+ or vortexName is None
326
+ )
327
+ ):
328
+ targetConns.append(conn)
329
+
330
+ if len(targetConns) > 1:
331
+ logger.warning(
332
+ "Multiple vortexes found for ip=%s, name=%s,"
333
+ " choosing latest connection",
334
+ ip,
335
+ vortexName,
336
+ )
337
+ targetConns.sort(key=lambda c: c.connectDateTime)
338
+ # As of Python3.6, dict are ordered, so the newest is the last.
339
+ return targetConns[-1].remoteVortexUuid
340
+
341
+ if not targetConns:
342
+ raise NoVortexException(f"Can not find vortex with IP {ip}")
343
+
344
+ return targetConns[0].remoteVortexUuid
345
+
346
+ @property
347
+ def getRemoteClientVortexInfos(self) -> List[VortexInfo]:
348
+ return [
349
+ VortexInfo(name=name, uuid=uuid)
350
+ for name, (vortex, uuid) in self.__remoteVortexUuidsByName.items()
351
+ ]
352
+
353
+ @property
354
+ def getRemoteVortexUuids(self) -> set[str]:
355
+ return self.__remoteVortexUuids
356
+
357
+ @property
358
+ def getRemoteVortexNames(self) -> list[str]:
359
+ return list(self.__remoteVortexUuidsByName)
360
+
361
+ @property
362
+ def getInboundConnectionCount(self) -> int:
363
+ return self.__inboundConnectionCount
364
+
365
+ def getInboundConnectionCountForPeer(self, peerIp: str) -> int:
366
+ return self.__inboundConnectionCountPerIp[peerIp]
367
+
368
+ def isVortexNameLocal(self, vortexName: str) -> bool:
369
+ return vortexName in self.__localVortexUuidsByName
370
+
371
+ def getLocalVortexClients(self, localVortexName: str) -> List[VortexABC]:
372
+ return [
373
+ vortex
374
+ for vortex, _ in self.__localVortexUuidsByName.get(
375
+ localVortexName, []
376
+ )
377
+ ]
378
+
379
+ def isRemoteVortexOnline(self, uuid):
380
+ return uuid in self.__remoteVortexNameByUuid
381
+
382
+ @property
383
+ def vortexSevers(self) -> tuple[VortexServer]:
384
+ return self.__vortexServers
385
+
386
+ @property
387
+ def vortexClients(self) -> tuple[VortexABC]:
388
+ return self.__vortexClients
389
+
390
+ @property
391
+ def allVortexes(self) -> tuple[VortexABC]:
392
+ return self.__allVortexes
393
+
394
+
395
+ class VortexFactory:
396
+ __vortexInfoState = _VortexFactoryVortexInfo()
397
+ __connectionRateLimit = _VortexConnectionRateLimit()
398
+
399
+ __vortexStatusChangeSubjectsByName: Dict[str, Subject] = {}
400
+
401
+ __isShutdown = False
402
+
403
+ __listeningPorts = []
404
+
405
+ _DEBUG_LOGGING = False
406
+
407
+ def __init__(self):
408
+ raise Exception("Vortex Factory should not be instantiated")
409
+
410
+ @classmethod
411
+ @inlineCallbacks
412
+ def shutdown(cls):
413
+ cls.__isShutdown = True
414
+ for vortex in cls.__vortexInfoState.allVortexes:
415
+ if hasattr(vortex, "close"):
416
+ vortex.close()
417
+ yield deferLater(reactor, 0.05, lambda: None)
418
+
419
+ while cls.__listeningPorts:
420
+ cls.__listeningPorts.pop().stopListening()
421
+ yield deferLater(reactor, 0.05, lambda: None)
422
+
423
+ cls.__vortexInfoState = _VortexFactoryVortexInfo()
424
+
425
+ @classmethod
426
+ def setPeerConnectionLimitPerIp(cls, limit: int):
427
+ cls.__connectionRateLimit.setPeerConnectionLimitPerIp(limit)
428
+
429
+ @classmethod
430
+ def setNewConnectionsPerWindow(cls, limit):
431
+ cls.__connectionRateLimit.setNewConnectionsPerWindow(limit)
432
+
433
+ @classmethod
434
+ def setRollingNewConnectionWindowSeconds(cls, seconds):
435
+ cls.__connectionRateLimit.setRollingNewConnectionWindowSeconds(seconds)
436
+
437
+ @classmethod
438
+ def createServer(cls, name: str, rootResource) -> None:
439
+ """Create Server
440
+
441
+ Create a vortex server, VortexServer clients connect to this vortex serer via HTTP(S)
442
+
443
+ VortexServer clients will connect and provide their names. This allows the factory to
444
+ abstract away the vortex UUIDs from their names.
445
+
446
+ :param name: The name of the local vortex.
447
+ :param rootResource: The resource to add the vortex to. An implementation of
448
+ C{twisted.web.resource.IResource}
449
+ :return: None
450
+ """
451
+
452
+ vortexServer = VortexServer(name)
453
+ cls.__vortexInfoState.addVortexServer(name, vortexServer)
454
+
455
+ vortexResource = VortexServerHttpResource(vortexServer)
456
+ rootResource.putChild(b"vortex", vortexResource)
457
+
458
+ @classmethod
459
+ def createWebsocketServer(
460
+ cls,
461
+ name: str,
462
+ port: int,
463
+ enableSsl: Optional[bool] = False,
464
+ sslEnableMutualTLS: Optional[bool] = False,
465
+ sslBundleFilePath: Optional[str] = None,
466
+ sslMutualTLSCertificateAuthorityBundleFilePath: Optional[str] = None,
467
+ sslMutualTLSTrustedPeerCertificateBundleFilePath: Optional[str] = None,
468
+ ) -> None:
469
+ """Create Server
470
+
471
+ Create a vortex server, VortexServer clients connect to this vortex serer via HTTP(S)
472
+
473
+ VortexServer clients will connect and provide their names. This allows the factory to
474
+ abstract away the vortex UUIDs from their names.
475
+
476
+ :param name: The name of the local vortex.
477
+ :param port: The tcp port to listen on
478
+ :param enableSsl: switch ssl on or off for HTTP
479
+ :param sslEnableMutualTLS: switch on or off mTLS
480
+ :param sslBundleFilePath: a filepath to a PEM file that contains
481
+ a private key, a certificate and a CA certificate
482
+ to identify the tls server itself
483
+ :param sslMutualTLSCertificateAuthorityBundleFilePath: a filepath to a
484
+ PEM file that contains all CA certificates which are
485
+ used for mutualTLS to verify the identities of the
486
+ tls clients
487
+ :param sslMutualTLSTrustedPeerCertificateBundleFilePath: a filepath that
488
+ loads all trusted peer certificates for mTLS when
489
+ `sslEnableMutualTLS` is enabled
490
+ :return: None
491
+ """
492
+ logger.info(
493
+ "Setting up Websocket Server '%s' at port %s, "
494
+ "with ssl '%s'"
495
+ "with client PEM bundle from '%s', "
496
+ "with mTLS '%s', "
497
+ "with mTLS CA bundle from '%s', "
498
+ "with mTLS trusted peer bundle from '%s'",
499
+ name,
500
+ port,
501
+ f"{'on' if enableSsl else 'off'}",
502
+ sslBundleFilePath,
503
+ f"{'on' if sslEnableMutualTLS else 'off'}",
504
+ sslMutualTLSCertificateAuthorityBundleFilePath,
505
+ sslMutualTLSTrustedPeerCertificateBundleFilePath,
506
+ )
507
+
508
+ vortexServer = VortexServer(name, requiresBase64Encoding=False)
509
+ cls.__vortexInfoState.addVortexServer(name, vortexServer)
510
+ vortexWebsocketServerFactory = VortexWebsocketServerFactory(
511
+ vortexServer
512
+ )
513
+ site = WebSocketFactory(vortexWebsocketServerFactory)
514
+ if enableSsl:
515
+ trustedCertificateAuthorities = None
516
+ if sslEnableMutualTLS:
517
+ trustedCertificateAuthorities = parseTrustRootFromBundle(
518
+ sslMutualTLSCertificateAuthorityBundleFilePath
519
+ )
520
+ trustedCertificateAuthorities = trustRootFromCertificates(
521
+ trustedCertificateAuthorities
522
+ )
523
+
524
+ privateKeyWithFullChain = parsePemBundleForServer(sslBundleFilePath)
525
+
526
+ trustedPeerCertificates = None
527
+ if sslMutualTLSTrustedPeerCertificateBundleFilePath is not None:
528
+ trustedPeerCertificates = parsePemBundleForTrustedPeers(
529
+ sslMutualTLSTrustedPeerCertificateBundleFilePath
530
+ )
531
+
532
+ sslContextFactory = buildCertificateOptionsForTwisted(
533
+ privateKeyWithFullChain,
534
+ trustRoot=trustedCertificateAuthorities,
535
+ trustedPeerCertificates=trustedPeerCertificates,
536
+ )
537
+ port = reactor.listenSSL(port, site, sslContextFactory)
538
+
539
+ else:
540
+ port = reactor.listenTCP(port, site)
541
+
542
+ cls.__listeningPorts.append(port)
543
+
544
+ @classmethod
545
+ def createHttpWebsocketResource(
546
+ cls, name: str
547
+ ) -> VortexWebSocketUpgradeResource:
548
+ vortexServer = VortexServer(name, requiresBase64Encoding=False)
549
+ cls.__vortexInfoState.addVortexServer(name, vortexServer)
550
+
551
+ vortexWebsocketServerFactory = VortexWebsocketServerFactory(
552
+ vortexServer
553
+ )
554
+ websocketFactory = VortexWrappedWebSocketFactory(
555
+ vortexWebsocketServerFactory
556
+ )
557
+
558
+ return VortexWebSocketUpgradeResource(websocketFactory)
559
+
560
+ @classmethod
561
+ def createHttpWebsocketServer(cls, name: str, rootResource) -> None:
562
+ """Create Server
563
+
564
+ Create a vortex server, VortexServer clients connect to this vortex serer via HTTP(S)
565
+
566
+ VortexServer clients will connect and provide their names. This allows the factory to
567
+ abstract away the vortex UUIDs from their names.
568
+
569
+ :param name: The name of the local vortex.
570
+ :param port: The tcp port to listen on
571
+ :return: None
572
+ """
573
+
574
+ vortexServer = VortexServer(name, requiresBase64Encoding=False)
575
+ cls.__vortexInfoState.addVortexServer(name, vortexServer)
576
+ vortexWebsocketServerFactory = VortexWebsocketServerFactory(
577
+ vortexServer
578
+ )
579
+ websocketFactory = VortexWrappedWebSocketFactory(
580
+ vortexWebsocketServerFactory
581
+ )
582
+
583
+ websocketResource = VortexWebSocketUpgradeResource(websocketFactory)
584
+ rootResource.putChild(b"vortexws", websocketResource)
585
+
586
+ @classmethod
587
+ def createTcpServer(cls, name: str, port: int) -> None:
588
+ """Create Server
589
+
590
+ Create a vortex server, VortexServer clients connect to this vortex serer via HTTP(S)
591
+
592
+ VortexServer clients will connect and provide their names. This allows the factory to
593
+ abstract away the vortex UUIDs from their names.
594
+
595
+ :param name: The name of the local vortex.
596
+ :param port: The tcp port to listen on
597
+ :return: None
598
+ """
599
+
600
+ vortexServer = VortexServer(name)
601
+ cls.__vortexInfoState.addVortexServer(name, vortexServer)
602
+ vortexTcpServerFactory = VortexTcpServerFactory(vortexServer)
603
+ port = reactor.listenTCP(port, vortexTcpServerFactory)
604
+ cls.__listeningPorts.append(port)
605
+
606
+ @classmethod
607
+ def createHttpClient(cls, name: str, host: str, port: int) -> Deferred:
608
+ """Create Client
609
+
610
+ Connect to a vortex Server.
611
+
612
+ :param name: The name of the local vortex.
613
+ :param host: The hostname of the remote vortex.
614
+ :param port: The port of the remote vortex.
615
+
616
+ :return: A deferred from the VortexHttpClient.connect method
617
+ """
618
+
619
+ logger.info("Connecting to Peek Server HTTP %s:%s", host, port)
620
+
621
+ vortexClient = VortexClientHttp(name)
622
+ cls.__vortexInfoState.addVortexClient(name, vortexClient)
623
+
624
+ return vortexClient.connect(host, port)
625
+
626
+ @classmethod
627
+ def createWebsocketClient(
628
+ cls,
629
+ name: str,
630
+ host: str,
631
+ port: int,
632
+ url: str,
633
+ sslEnableMutualTLS: bool = False,
634
+ sslClientCertificateBundleFilePath: str = None,
635
+ sslMutualTLSCertificateAuthorityBundleFilePath: str = None,
636
+ sslMutualTLSTrustedPeerCertificateBundleFilePath: str = None,
637
+ ) -> Deferred:
638
+ """Create websocket client
639
+
640
+ Connect to a vortex websocket server
641
+ :param name: The name of the local vortex
642
+ :param host: The hostname of the remote vortex
643
+ :param port: The port of the remote vortex
644
+ :param url: The websocket url that this client tries to connect to
645
+ :param sslEnableMutualTLS: switch on or off mTLS
646
+ :param sslClientCertificateBundleFilePath: a PEM bundle file that
647
+ contains a pair of key and certificate for this web client
648
+ :param sslMutualTLSCertificateAuthorityBundleFilePath:CA bundle file path
649
+ for TLS client authentication
650
+
651
+ :param sslMutualTLSTrustedPeerCertificateBundleFilePath: a PEM bundle
652
+ file that contains certificates of all trusted peers. Each
653
+ certificate means a PEM with no intermediate CAs and/or
654
+ root CAs.
655
+ :return: A deferred from the autobahn.twisted.connectWS method
656
+ """
657
+ logger.info(
658
+ "Connecting to Peek Websocket Server %s:%s, "
659
+ "with client PEM bundle from '%s', "
660
+ "with mTLS '%s', "
661
+ "with mTLS CA bundle from '%s', "
662
+ "with mTLS trusted peer bundle from '%s' ",
663
+ host,
664
+ port,
665
+ sslClientCertificateBundleFilePath,
666
+ f"{'on' if sslEnableMutualTLS else 'off'}",
667
+ sslMutualTLSCertificateAuthorityBundleFilePath,
668
+ sslMutualTLSTrustedPeerCertificateBundleFilePath,
669
+ )
670
+ vortexWebsocketClientFactory = VortexClientWebsocketFactory(
671
+ name, url=url
672
+ )
673
+ cls.__vortexInfoState.addVortexClient(
674
+ name, vortexWebsocketClientFactory
675
+ )
676
+
677
+ if vortexWebsocketClientFactory.isSecure and sslEnableMutualTLS:
678
+ trustedCertificateAuthorities = parseTrustRootFromBundle(
679
+ sslMutualTLSCertificateAuthorityBundleFilePath
680
+ )
681
+ trustedCertificateAuthorities = trustRootFromCertificates(
682
+ trustedCertificateAuthorities
683
+ )
684
+
685
+ privateKeyWithFullChain = parsePemBundleForClient(
686
+ sslClientCertificateBundleFilePath
687
+ )
688
+
689
+ trustedPeerCertificates = None
690
+ if sslMutualTLSTrustedPeerCertificateBundleFilePath is not None:
691
+ trustedPeerCertificates = parsePemBundleForTrustedPeers(
692
+ sslMutualTLSTrustedPeerCertificateBundleFilePath
693
+ )
694
+
695
+ sslContextFactory = buildCertificateOptionsForTwisted(
696
+ privateKeyWithFullChain,
697
+ trustRoot=trustedCertificateAuthorities,
698
+ trustedPeerCertificates=trustedPeerCertificates,
699
+ )
700
+ else:
701
+ # use default for http or normal https
702
+ sslContextFactory = ssl.ClientContextFactory()
703
+ # raise minimum version to tls 1.2
704
+ sslContextFactory.method = SSL.TLSv1_2_METHOD
705
+
706
+ return vortexWebsocketClientFactory.connect(
707
+ host, port, sslContextFactory
708
+ )
709
+
710
+ @classmethod
711
+ def createTcpClient(cls, name: str, host: str, port: int) -> Deferred:
712
+ """Create Client
713
+
714
+ Connect to a vortex Server.
715
+
716
+ :param name: The name of the local vortex.
717
+ :param host: The hostname of the remote vortex.
718
+ :param port: The port of the remote vortex.
719
+
720
+ :return: A deferred from the VortexTcpClient.connect method
721
+ """
722
+
723
+ logger.info("Connecting to Peek Server TCP %s:%s", host, port)
724
+
725
+ vortexClient = VortexClientTcp(name)
726
+ cls.__vortexInfoState.addVortexClient(name, vortexClient)
727
+
728
+ return vortexClient.connect(host, port)
729
+
730
+ @classmethod
731
+ def addCustomServerVortex(cls, vortex: VortexABC):
732
+ cls.__vortexInfoState.addVortexServer(
733
+ vortex.localVortexInfo.name, vortex
734
+ )
735
+
736
+ @classmethod
737
+ def canConnect(cls, fromIp: str) -> bool:
738
+ return cls.__connectionRateLimit.canConnect(
739
+ cls.__vortexInfoState.getInboundConnectionCountForPeer(fromIp),
740
+ cls.__vortexInfoState.getInboundConnectionCount,
741
+ fromIp,
742
+ )
743
+
744
+ @classmethod
745
+ def isVortexUuidOnline(cls, vortexUuid: str) -> bool:
746
+ return cls.__vortexInfoState.isRemoteVortexOnline(uuid=vortexUuid)
747
+
748
+ @classmethod
749
+ def isVortexNameLocal(cls, vortexName: str) -> bool:
750
+ return cls.__vortexInfoState.isVortexNameLocal(vortexName=vortexName)
751
+
752
+ @classmethod
753
+ def getLocalVortexClients(cls, localVortexName: str) -> List[VortexABC]:
754
+ return cls.__vortexInfoState.getLocalVortexClients(
755
+ localVortexName=localVortexName
756
+ )
757
+
758
+ @classmethod
759
+ def getRemoteClientVortexInfos(cls) -> List[VortexInfo]:
760
+ return cls.__vortexInfoState.getRemoteClientVortexInfos
761
+
762
+ @classmethod
763
+ def getRemoteVortexUuids(cls) -> set[str]:
764
+ return cls.__vortexInfoState.getRemoteVortexUuids
765
+
766
+ @classmethod
767
+ def getInboundConnectionCount(cls) -> int:
768
+ return cls.__vortexInfoState.getInboundConnectionCount
769
+
770
+ @classmethod
771
+ def getRemoteVortexName(cls) -> list[str]:
772
+ return cls.__vortexInfoState.getRemoteVortexNames
773
+
774
+ @classmethod
775
+ def getRemoteVortexIpByUuid(cls, vortexUuid: str, vortexName=None) -> str:
776
+ return cls.__vortexInfoState.getRemoteVortexIpByUuid(
777
+ vortexUuid=vortexUuid, vortexName=vortexName
778
+ )
779
+
780
+ @classmethod
781
+ def getRemoteVortexInfoByIp(cls, ip: str, vortexName=None) -> str:
782
+ return cls.__vortexInfoState.getRemoteVortexUuidByIp(
783
+ ip=ip, vortexName=vortexName
784
+ )
785
+
786
+ @classmethod
787
+ def getRemoteVortexUuidByIp(cls, ip: str, vortexName=None) -> str:
788
+ # alias to getRemoteVortexInfoByIp
789
+ return cls.getRemoteVortexInfoByIp(ip=ip, vortexName=vortexName)
790
+
791
+ @classmethod
792
+ @inlineCallbacks
793
+ def sendVortexMsg(
794
+ cls,
795
+ vortexMsgs: Union[VortexMsgList, bytes],
796
+ destVortexName: Optional[str] = broadcast,
797
+ destVortexUuid: Optional[str] = broadcast,
798
+ ) -> Deferred:
799
+ """Send VortexMsg
800
+
801
+ Sends a payload to the remote vortex.
802
+
803
+ :param vortexMsgs: The vortex message(s) to send to the remote vortex.
804
+
805
+ :param destVortexName: The name of the vortex to send the payload to.
806
+ This can be null, If provided, it's used to limit the vortexes the
807
+ message is sent to.
808
+
809
+ :param destVortexUuid: The uuid of the vortex to send the payload to,
810
+ This can be null, If provided, it's used to limit the vortexes the
811
+ message is sent to.
812
+
813
+
814
+ :return: A C{Deferred} which will callback when the message has been sent.
815
+ """
816
+ vortexAndUuids = cls.__vortexInfoState.getRemoteVortexInfos(
817
+ name=destVortexName, uuid=destVortexUuid
818
+ )
819
+ if not vortexAndUuids:
820
+ raise NoVortexException(
821
+ "Can not find vortexes to send message to,"
822
+ " name=%s, uuid=%s" % (destVortexName, destVortexUuid)
823
+ )
824
+
825
+ if not isinstance(vortexMsgs, list):
826
+ vortexMsgs = [vortexMsgs]
827
+
828
+ base64VortexMsgs = vortexMsgs[:]
829
+
830
+ # If any transports require base64 encoding, then encode them all
831
+
832
+ if [v for v in vortexAndUuids if v[0].requiresBase64Encoding]:
833
+ for index, vortexMsg in enumerate(base64VortexMsgs):
834
+ if vortexMsg.startswith(b"{"):
835
+ base64VortexMsg = yield PayloadEnvelope.base64EncodeDefer(
836
+ vortexMsg
837
+ )
838
+ base64VortexMsgs[index] = base64VortexMsg + b"."
839
+
840
+ deferreds = []
841
+ for vortex, uuid in vortexAndUuids:
842
+ msgToUse = (
843
+ base64VortexMsgs
844
+ if vortex.requiresBase64Encoding
845
+ else vortexMsgs
846
+ )
847
+ deferreds.append(vortex.sendVortexMsg(msgToUse, uuid))
848
+
849
+ results = yield DeferredList(deferreds)
850
+ return results
851
+
852
+ @classmethod
853
+ def sendVortexMsgLocally(
854
+ cls,
855
+ vortexMsgs: Union[VortexMsgList, bytes],
856
+ priority: int = DEFAULT_PRIORITY,
857
+ ) -> Deferred:
858
+ """Send VortexMsg
859
+
860
+ Sends a payload to the remote vortex.
861
+
862
+ :param vortexMsgs: The vortex message(s) to deliver locally.
863
+
864
+ :return: A C{Deferred} which will callback when the message has been delivered.
865
+ """
866
+ yesMainThread()
867
+
868
+ vortexUuid = "local"
869
+ vortexName = "local"
870
+ httpSession = "local"
871
+ sendResponse = VortexFactory.sendVortexMsgLocally
872
+
873
+ vortexMsgs = (
874
+ [vortexMsgs] if isinstance(vortexMsgs, bytes) else vortexMsgs
875
+ )
876
+
877
+ def send(payloadEnvelope: PayloadEnvelope):
878
+ try:
879
+ PayloadIO().process(
880
+ payloadEnvelope=payloadEnvelope,
881
+ vortexUuid=vortexUuid,
882
+ vortexName=vortexName,
883
+ httpSession=httpSession,
884
+ sendResponse=sendResponse,
885
+ )
886
+ return succeed(True)
887
+
888
+ except Exception as e:
889
+ return Failure(e)
890
+
891
+ deferreds = []
892
+ for vortexMsg in vortexMsgs:
893
+ d = PayloadEnvelope().fromVortexMsgDefer(vortexMsg)
894
+ d.addCallback(send)
895
+ deferreds.append(d)
896
+
897
+ return DeferredList(deferreds)
898
+
899
+ @classmethod
900
+ def connectionChanged(cls):
901
+ cls.__vortexInfoState.rebuildStructs()
902
+
903
+ @classmethod
904
+ def subscribeToVortexStatusChange(cls, vortexName: str) -> Subject:
905
+ """Subscribe to Vortex Status Change
906
+
907
+ Subscribing to the returned observable/subject will provided updates of when the
908
+ vortex goes offline, or online.
909
+
910
+ .. warning:: This is only implemented for TCP Client vortexes.
911
+
912
+ :param vortexName: The name of the vortex to subscribe to the status for.
913
+ This will be the name of the remote vortex.
914
+
915
+ """
916
+ if not vortexName in cls.__vortexStatusChangeSubjectsByName:
917
+ cls.__vortexStatusChangeSubjectsByName[vortexName] = Subject()
918
+ return cls.__vortexStatusChangeSubjectsByName[vortexName]
919
+
920
+ @classmethod
921
+ def _notifyOfVortexStatusChange(cls, vortexName: str, online: bool) -> None:
922
+ if cls.__isShutdown:
923
+ return
924
+
925
+ logger.debug(
926
+ "Vortex %s went %s", vortexName, ("online" if online else "offline")
927
+ )
928
+
929
+ if vortexName in cls.__vortexStatusChangeSubjectsByName:
930
+ cls.__vortexStatusChangeSubjectsByName[vortexName].on_next(online)
931
+
932
+ cls.connectionChanged()