dimples 0.3.4__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {dimples-0.3.4 → dimples-0.4.1}/PKG-INFO +1 -1
  2. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/network/state.py +1 -1
  3. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/terminal.py +5 -0
  4. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/gate.py +22 -20
  5. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/gatekeeper.py +1 -1
  6. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/mars.py +22 -10
  7. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/mtp.py +2 -0
  8. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/protocol/ws.py +35 -18
  9. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/ws.py +23 -11
  10. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/__init__.py +0 -4
  11. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/archivist.py +73 -11
  12. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/processor.py +44 -13
  13. {dimples-0.3.4 → dimples-0.4.1}/dimples/station/shared.py +0 -9
  14. {dimples-0.3.4 → dimples-0.4.1}/dimples.egg-info/PKG-INFO +1 -1
  15. {dimples-0.3.4 → dimples-0.4.1}/dimples.egg-info/SOURCES.txt +0 -1
  16. {dimples-0.3.4 → dimples-0.4.1}/dimples.egg-info/requires.txt +3 -3
  17. {dimples-0.3.4 → dimples-0.4.1}/setup.py +4 -4
  18. dimples-0.3.4/dimples/server/broadcast.py +0 -219
  19. {dimples-0.3.4 → dimples-0.4.1}/README.md +0 -0
  20. {dimples-0.3.4 → dimples-0.4.1}/dimples/__init__.py +0 -0
  21. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/__init__.py +0 -0
  22. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/archivist.py +0 -0
  23. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/checkpoint.py +0 -0
  24. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/__init__.py +0 -0
  25. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/commands.py +0 -0
  26. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/creator.py +0 -0
  27. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/group.py +0 -0
  28. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_expel.py +0 -0
  29. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_invite.py +0 -0
  30. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_join.py +0 -0
  31. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_query.py +0 -0
  32. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_quit.py +0 -0
  33. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_reset.py +0 -0
  34. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/grp_resign.py +0 -0
  35. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/cpu/handshake.py +0 -0
  36. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/facebook.py +0 -0
  37. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/messenger.py +0 -0
  38. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/network/__init__.py +0 -0
  39. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/network/session.py +0 -0
  40. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/network/transition.py +0 -0
  41. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/packer.py +0 -0
  42. {dimples-0.3.4 → dimples-0.4.1}/dimples/client/processor.py +0 -0
  43. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/__init__.py +0 -0
  44. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/anonymous.py +0 -0
  45. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/ans.py +0 -0
  46. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/archivist.py +0 -0
  47. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/compat/__init__.py +0 -0
  48. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/compat/btc.py +0 -0
  49. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/compat/compatible.py +0 -0
  50. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/compat/entity.py +0 -0
  51. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/compat/meta.py +0 -0
  52. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/compat/network.py +0 -0
  53. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/dbi/__init__.py +0 -0
  54. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/dbi/account.py +0 -0
  55. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/dbi/message.py +0 -0
  56. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/dbi/session.py +0 -0
  57. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/facebook.py +0 -0
  58. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/messenger.py +0 -0
  59. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/packer.py +0 -0
  60. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/__init__.py +0 -0
  61. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/ans.py +0 -0
  62. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/block.py +0 -0
  63. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/handshake.py +0 -0
  64. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/login.py +0 -0
  65. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/mute.py +0 -0
  66. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/protocol/report.py +0 -0
  67. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/register.py +0 -0
  68. {dimples-0.3.4 → dimples-0.4.1}/dimples/common/session.py +0 -0
  69. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/__init__.py +0 -0
  70. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/protocol/__init__.py +0 -0
  71. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/protocol/mars.py +0 -0
  72. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/queue.py +0 -0
  73. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/seeker.py +0 -0
  74. {dimples-0.3.4 → dimples-0.4.1}/dimples/conn/session.py +0 -0
  75. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/__init__.py +0 -0
  76. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/account.py +0 -0
  77. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/__init__.py +0 -0
  78. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/base.py +0 -0
  79. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/document.py +0 -0
  80. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/group.py +0 -0
  81. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/group_history.py +0 -0
  82. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/group_keys.py +0 -0
  83. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/login.py +0 -0
  84. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/meta.py +0 -0
  85. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/private.py +0 -0
  86. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/station.py +0 -0
  87. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/dos/user.py +0 -0
  88. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/message.py +0 -0
  89. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/session.py +0 -0
  90. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_cipherkey.py +0 -0
  91. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_document.py +0 -0
  92. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_group.py +0 -0
  93. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_group_history.py +0 -0
  94. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_group_keys.py +0 -0
  95. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_login.py +0 -0
  96. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_message.py +0 -0
  97. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_meta.py +0 -0
  98. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_private.py +0 -0
  99. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_station.py +0 -0
  100. {dimples-0.3.4 → dimples-0.4.1}/dimples/database/t_user.py +0 -0
  101. {dimples-0.3.4 → dimples-0.4.1}/dimples/edge/__init__.py +0 -0
  102. {dimples-0.3.4 → dimples-0.4.1}/dimples/edge/octopus.py +0 -0
  103. {dimples-0.3.4 → dimples-0.4.1}/dimples/edge/shared.py +0 -0
  104. {dimples-0.3.4 → dimples-0.4.1}/dimples/edge/start.py +0 -0
  105. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/__init__.py +0 -0
  106. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/admin.py +0 -0
  107. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/builder.py +0 -0
  108. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/delegate.py +0 -0
  109. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/emitter.py +0 -0
  110. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/helper.py +0 -0
  111. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/manager.py +0 -0
  112. {dimples-0.3.4 → dimples-0.4.1}/dimples/group/packer.py +0 -0
  113. {dimples-0.3.4 → dimples-0.4.1}/dimples/register/__init__.py +0 -0
  114. {dimples-0.3.4 → dimples-0.4.1}/dimples/register/base.py +0 -0
  115. {dimples-0.3.4 → dimples-0.4.1}/dimples/register/ext.py +0 -0
  116. {dimples-0.3.4 → dimples-0.4.1}/dimples/register/run.py +0 -0
  117. {dimples-0.3.4 → dimples-0.4.1}/dimples/register/shared.py +0 -0
  118. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/cpu/__init__.py +0 -0
  119. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/cpu/ans.py +0 -0
  120. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/cpu/document.py +0 -0
  121. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/cpu/handshake.py +0 -0
  122. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/cpu/login.py +0 -0
  123. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/cpu/report.py +0 -0
  124. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/dispatcher.py +0 -0
  125. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/messenger.py +0 -0
  126. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/packer.py +0 -0
  127. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/push.py +0 -0
  128. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/session.py +0 -0
  129. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/session_center.py +0 -0
  130. {dimples-0.3.4 → dimples-0.4.1}/dimples/server/trace.py +0 -0
  131. {dimples-0.3.4 → dimples-0.4.1}/dimples/station/__init__.py +0 -0
  132. {dimples-0.3.4 → dimples-0.4.1}/dimples/station/handler.py +0 -0
  133. {dimples-0.3.4 → dimples-0.4.1}/dimples/station/start.py +0 -0
  134. {dimples-0.3.4 → dimples-0.4.1}/dimples/utils/__init__.py +0 -0
  135. {dimples-0.3.4 → dimples-0.4.1}/dimples/utils/cache.py +0 -0
  136. {dimples-0.3.4 → dimples-0.4.1}/dimples/utils/config.py +0 -0
  137. {dimples-0.3.4 → dimples-0.4.1}/dimples/utils/dos.py +0 -0
  138. {dimples-0.3.4 → dimples-0.4.1}/dimples/utils/log.py +0 -0
  139. {dimples-0.3.4 → dimples-0.4.1}/dimples/utils/singleton.py +0 -0
  140. {dimples-0.3.4 → dimples-0.4.1}/dimples.egg-info/dependency_links.txt +0 -0
  141. {dimples-0.3.4 → dimples-0.4.1}/dimples.egg-info/entry_points.txt +0 -0
  142. {dimples-0.3.4 → dimples-0.4.1}/dimples.egg-info/top_level.txt +0 -0
  143. {dimples-0.3.4 → dimples-0.4.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dimples
3
- Version: 0.3.4
3
+ Version: 0.4.1
4
4
  Summary: DIMP Library for Edges and Stations
5
5
  Home-page: https://github.com/dimchat/demo-py
6
6
  Author: Albert Moky
@@ -79,7 +79,7 @@ class StateMachine(AutoMachine, Context):
79
79
  def status(self) -> DockerStatus:
80
80
  session = self.session
81
81
  gate = session.gate
82
- docker = gate.get_docker(remote=session.remote_address, local=None, advance_party=[])
82
+ docker = gate.fetch_docker(remote=session.remote_address, local=None, advance_party=[])
83
83
  if docker is None:
84
84
  return DockerStatus.ERROR
85
85
  else:
@@ -149,6 +149,11 @@ class Terminal(Runner, StateDelegate, Logging):
149
149
  # broadcast current meta & visa document to all stations
150
150
  messenger = self.messenger
151
151
  messenger.handshake_success()
152
+ session = messenger.session
153
+ usr_id = session.identifier
154
+ if usr_id is not None and usr_id.type != EntityType.STATION:
155
+ # send login command to everyone to provide more information.
156
+ messenger.broadcast_login(sender=usr_id, user_agent=self.user_agent)
152
157
  # update last online time
153
158
  self.__last_time = time.time()
154
159
 
@@ -30,8 +30,9 @@
30
30
 
31
31
  import socket
32
32
  from abc import ABC
33
- from typing import Generic, TypeVar, Optional, Union, List, Tuple
33
+ from typing import Generic, TypeVar, Optional, Union, List
34
34
 
35
+ from startrek.types import SocketAddress
35
36
  from startrek import Hub, Channel
36
37
  from startrek import Connection, ConnectionState, ActiveConnection
37
38
  from startrek import Docker, DockerDelegate
@@ -65,30 +66,17 @@ class BaseGate(StarGate, Generic[H], ABC):
65
66
  #
66
67
  # Docker
67
68
  #
68
- def get_docker(self, remote: Tuple[str, int], local: Optional[Tuple[str, int]],
69
- advance_party: List[bytes]) -> Docker:
70
- docker = self._get_docker(remote=remote, local=local)
71
- if docker is None:
72
- hub = self.hub
73
- # from startrek import Hub
74
- # assert isinstance(hub, Hub)
75
- conn = hub.connect(remote=remote, local=local)
76
- if conn is not None:
77
- docker = self._create_docker(connection=conn, advance_party=advance_party)
78
- assert docker is not None, 'failed to create docker: %s, %s' % (remote, local)
79
- self._set_docker(remote=remote, local=local, docker=docker)
80
- return docker
81
69
 
82
70
  # Override
83
- def _get_docker(self, remote: Tuple[str, int], local: Optional[Tuple[str, int]]) -> Optional[Docker]:
71
+ def _get_docker(self, remote: SocketAddress, local: Optional[SocketAddress]) -> Optional[Docker]:
84
72
  return super()._get_docker(remote=remote, local=None)
85
73
 
86
74
  # Override
87
- def _set_docker(self, remote: Tuple[str, int], local: Optional[Tuple[str, int]], docker: Docker):
75
+ def _set_docker(self, remote: SocketAddress, local: Optional[SocketAddress], docker: Docker):
88
76
  super()._set_docker(remote=remote, local=None, docker=docker)
89
77
 
90
78
  # Override
91
- def _remove_docker(self, remote: Tuple[str, int], local: Optional[Tuple[str, int]], docker: Optional[Docker]):
79
+ def _remove_docker(self, remote: SocketAddress, local: Optional[SocketAddress], docker: Optional[Docker]):
92
80
  super()._remove_docker(remote=remote, local=None, docker=docker)
93
81
 
94
82
  # Override
@@ -171,14 +159,14 @@ class CommonGate(BaseGate, Logging, Generic[H], ABC):
171
159
  if error is not None and str(error).startswith('failed to send: '):
172
160
  self.warning(msg='ignore socket error: %s, remote=%s' % (error, connection.remote_address))
173
161
 
174
- def get_channel(self, remote: Tuple[str, int], local: Optional[Tuple[str, int]]) -> Optional[Channel]:
162
+ def get_channel(self, remote: Optional[SocketAddress], local: Optional[SocketAddress]) -> Optional[Channel]:
175
163
  hub = self.hub
176
164
  assert isinstance(hub, Hub), 'hub error: %s' % hub
177
165
  return hub.open(remote=remote, local=local)
178
166
 
179
167
  def send_response(self, payload: bytes, ship: Arrival,
180
- remote: Tuple[str, int], local: Optional[Tuple[str, int]]) -> bool:
181
- worker = self.get_docker(remote=remote, local=local, advance_party=[])
168
+ remote: SocketAddress, local: Optional[SocketAddress]) -> bool:
169
+ worker = self._get_docker(remote=remote, local=local)
182
170
  if isinstance(worker, MTPStreamDocker):
183
171
  # sn = TransactionID.from_data(data=ship.sn)
184
172
  sn = TransactionID.generate()
@@ -195,6 +183,20 @@ class CommonGate(BaseGate, Logging, Generic[H], ABC):
195
183
  else:
196
184
  raise LookupError('docker error (%s, %s): %s' % (remote, local, worker))
197
185
 
186
+ def fetch_docker(self, remote: SocketAddress, local: Optional[SocketAddress], advance_party: List[bytes]) -> Docker:
187
+ docker = self._get_docker(remote=remote, local=local)
188
+ if docker is None: # and advance_party is not None:
189
+ hub = self.hub
190
+ assert isinstance(hub, Hub), 'gate hub error: %s' % hub
191
+ conn = hub.connect(remote=remote, local=local)
192
+ if conn is not None:
193
+ docker = self._create_docker(connection=conn, advance_party=advance_party)
194
+ if docker is None:
195
+ assert False, 'failed to create docker: %s, %s' % (remote, local)
196
+ else:
197
+ self._set_docker(remote=remote, local=local, docker=docker)
198
+ return docker
199
+
198
200
 
199
201
  #
200
202
  # Server Gates
@@ -236,7 +236,7 @@ class GateKeeper(Runner, DockerDelegate, Logging):
236
236
  return ok
237
237
 
238
238
  def _docker_pack(self, payload: bytes, priority: int = 0) -> Departure:
239
- docker = self.gate.get_docker(remote=self.remote_address, local=None, advance_party=[])
239
+ docker = self.gate.fetch_docker(remote=self.remote_address, local=None, advance_party=[])
240
240
  assert isinstance(docker, DeparturePacker), 'departure packer error: %s' % docker
241
241
  return docker.pack(payload=payload, priority=priority)
242
242
 
@@ -189,7 +189,7 @@ class MarsStreamDocker(PlainDocker, DeparturePacker):
189
189
  self.__chunks_lock = threading.RLock()
190
190
  self.__package_received = False
191
191
 
192
- def _parse_package(self, data: bytes) -> Optional[NetMsg]:
192
+ def _parse_package(self, data: bytes) -> Tuple[Optional[NetMsg], int]:
193
193
  with self.__chunks_lock:
194
194
  # join the data to the memory cache
195
195
  data = self.__chunks + data
@@ -197,16 +197,18 @@ class MarsStreamDocker(PlainDocker, DeparturePacker):
197
197
  # try to fetch a package
198
198
  pack, offset = MarsHelper.seek_package(data=data)
199
199
  self.__package_received = pack is not None
200
+ remain_len = len(data)
200
201
  if offset >= 0:
201
202
  # 'error part' + 'mars package' + 'remaining data
202
203
  if pack is not None:
203
204
  offset += pack.length
204
205
  if offset == 0:
205
206
  self.__chunks = data + self.__chunks
206
- elif offset < len(data):
207
+ elif offset < remain_len:
207
208
  data = data[offset:]
208
209
  self.__chunks = data + self.__chunks
209
- return pack
210
+ remain_len -= offset
211
+ return pack, remain_len
210
212
 
211
213
  # Override
212
214
  def process_received(self, data: bytes):
@@ -219,13 +221,23 @@ class MarsStreamDocker(PlainDocker, DeparturePacker):
219
221
  data = b''
220
222
 
221
223
  # Override
222
- def _get_arrival(self, data: bytes) -> Optional[Arrival]:
223
- pack = self._parse_package(data=data)
224
- if pack is None:
225
- return None
226
- # if pack.body is None:
227
- # return None
228
- return MarsStreamArrival(mars=pack)
224
+ def _get_arrivals(self, data: bytes) -> List[Arrival]:
225
+ ships = []
226
+ while True:
227
+ pack, remain_len = self._parse_package(data=data)
228
+ if pack is None:
229
+ # waiting for more data
230
+ break
231
+ # if pack.body is None:
232
+ # continue
233
+ ships.append(MarsStreamArrival(mars=pack))
234
+ if remain_len > 0:
235
+ # continue to check the tail
236
+ data = b''
237
+ else:
238
+ # all data processed
239
+ break
240
+ return ships
229
241
 
230
242
  # Override
231
243
  def _check_arrival(self, ship: Arrival) -> Optional[Arrival]:
@@ -121,6 +121,7 @@ class MTPStreamDocker(PackageDocker, DeparturePacker):
121
121
  # try to fetch a package
122
122
  pack, offset = MTPHelper.seek_package(data=data)
123
123
  self.__package_received = pack is not None
124
+ remain_len = len(data)
124
125
  if offset >= 0:
125
126
  # 'error part' + 'MTP package' + 'remaining data
126
127
  if pack is not None:
@@ -130,6 +131,7 @@ class MTPStreamDocker(PackageDocker, DeparturePacker):
130
131
  elif offset < data.size:
131
132
  data = data.slice(start=offset)
132
133
  self.__chunks = data.concat(self.__chunks)
134
+ remain_len -= offset
133
135
  return pack
134
136
 
135
137
  # Override
@@ -34,11 +34,11 @@
34
34
 
35
35
  """
36
36
 
37
- import hashlib
38
37
  import struct
39
38
  from typing import Optional, Tuple
40
39
 
41
- from ...utils import base64_encode, utf8_encode
40
+ from ...utils import sha1, base64_encode, utf8_encode, utf8_decode
41
+ from ...utils import Log
42
42
 
43
43
 
44
44
  class WebSocket:
@@ -57,22 +57,35 @@ class WebSocket:
57
57
 
58
58
  @classmethod
59
59
  def handshake(cls, stream: bytes) -> Optional[bytes]:
60
- pos1 = stream.find(b'Sec-WebSocket-Key:')
61
- if pos1 < 0:
60
+ key = cls.__fetch_key(stream=stream)
61
+ if key is None:
62
62
  return None
63
- pos1 += len('Sec-WebSocket-Key:')
64
- pos2 = stream.find(b'\r\n', pos1)
65
- if pos2 < 0:
66
- return None
67
- key = stream[pos1:pos2].strip()
68
- sec = hashlib.sha1(key + cls.ws_magic).digest()
63
+ # build response with sec-key
64
+ sec = sha1(key + cls.ws_magic)
69
65
  sec = base64_encode(data=sec)
70
66
  sec = utf8_encode(string=sec)
71
67
  return cls.ws_prefix + sec + cls.ws_suffix
72
68
 
73
69
  @classmethod
74
70
  def is_handshake(cls, stream: bytes) -> bool:
75
- return stream.find(b'Sec-WebSocket-Key') > 0
71
+ return cls.__fetch_key(stream=stream) is not None
72
+
73
+ @classmethod
74
+ def __fetch_key(cls, stream: bytes) -> Optional[bytes]:
75
+ if cls.__check_http(stream=stream):
76
+ text = utf8_decode(data=stream).lower()
77
+ pos1 = text.find('sec-websocket-key:')
78
+ if pos1 > 0:
79
+ pos1 += len('sec-websocket-key:')
80
+ pos2 = text.find('\r\n', pos1)
81
+ if pos2 > 0:
82
+ return stream[pos1:pos2].strip()
83
+
84
+ @classmethod
85
+ def __check_http(cls, stream: bytes) -> bool:
86
+ if stream.startswith(b'GET /'):
87
+ pos = stream.find(b'HTTP/')
88
+ return 5 < pos < 512
76
89
 
77
90
  """
78
91
  RFC: https://tools.ietf.org/html/rfc6455#section-5.2
@@ -105,13 +118,14 @@ class WebSocket:
105
118
  :return: (payload, remaining_data)
106
119
  """
107
120
  stream_len = len(stream)
121
+ print('parsing stream: %d bytes' % stream_len)
108
122
  if stream_len < 2:
109
123
  return None, stream
110
124
  data = b''
111
125
  pos = 0
112
126
  while True:
113
127
  if stream_len < pos + 2:
114
- # self.info(msg='incomplete ws package for op code: %d' % stream_len)
128
+ Log.info(msg='incomplete ws package for op code: %d' % stream_len)
115
129
  return None, stream
116
130
  # 1. check whether a continuation frame
117
131
  ch0 = stream[pos+0]
@@ -125,7 +139,7 @@ class WebSocket:
125
139
  msg_len = ch1 & 0x7F
126
140
  if msg_len == 126:
127
141
  if stream_len < pos + 4:
128
- # self.info(msg='incomplete ws package for msg len: %d' % stream_len)
142
+ Log.info(msg='incomplete ws package for msg len: %d' % stream_len)
129
143
  return None, stream
130
144
  b2 = stream[pos+2]
131
145
  b3 = stream[pos+3]
@@ -133,7 +147,7 @@ class WebSocket:
133
147
  pos += 4
134
148
  elif msg_len == 127:
135
149
  if stream_len < pos + 10:
136
- # self.info(msg='incomplete ws package for msg len: %d' % stream_len)
150
+ Log.info(msg='incomplete ws package for msg len: %d' % stream_len)
137
151
  return None, stream
138
152
  b2 = stream[pos+2]
139
153
  b3 = stream[pos+3]
@@ -150,7 +164,7 @@ class WebSocket:
150
164
  # 3. get masking-key
151
165
  if mask == 1:
152
166
  if stream_len < pos + 4:
153
- # self.info(msg='incomplete ws package for mask: %d' % stream_len)
167
+ Log.info(msg='incomplete ws package for mask: %d' % stream_len)
154
168
  return None, stream
155
169
  mask = stream[pos:pos+4]
156
170
  pos += 4
@@ -158,7 +172,7 @@ class WebSocket:
158
172
  mask = None
159
173
  # 4. get payload
160
174
  if stream_len < pos + msg_len:
161
- # self.info(msg='incomplete ws package for payload: %d' % stream_len)
175
+ Log.info(msg='incomplete ws package for payload: %d' % stream_len)
162
176
  return None, stream
163
177
  payload = stream[pos:pos+msg_len]
164
178
  pos += msg_len
@@ -179,22 +193,25 @@ class WebSocket:
179
193
  data += content
180
194
  elif op == 8:
181
195
  # TODO: CLOSE
196
+ Log.warning(msg='CLOSE')
182
197
  pass
183
198
  elif op == 9:
184
199
  # TODO: PING
200
+ Log.warning(msg='PING')
185
201
  pass
186
202
  elif op == 10:
187
203
  # TODO: PONG
204
+ Log.warning(msg='PONG')
188
205
  pass
189
206
  else:
190
- # self.error(msg='ws op error: %d => %s' % (op, stream))
207
+ Log.error(msg='ws op error: %d => %s' % (op, stream))
191
208
  return None, b''
192
209
  # 6. check final fragment
193
210
  if fin == 1 or op == 0:
194
211
  # cut the received package(s) and return the remaining
195
212
  stream = stream[pos:]
196
213
  break
197
- # self.info(msg='received ws payload len: %d' % len(data))
214
+ Log.info(msg='received ws payload len: %d, left: %d' % (len(data), len(stream)))
198
215
  return data, stream
199
216
 
200
217
  @classmethod
@@ -114,24 +114,25 @@ class WSDocker(PlainDocker, DeparturePacker):
114
114
  self.__chunks_lock = threading.RLock()
115
115
  self.__package_received = False
116
116
 
117
- def _parse_package(self, data: bytes) -> Tuple[Optional[bytes], Optional[bytes]]:
117
+ def _parse_package(self, data: bytes) -> Tuple[Optional[bytes], Optional[bytes], int]:
118
118
  with self.__chunks_lock:
119
119
  # join the data to the memory cache
120
120
  data = self.__chunks + data
121
121
  self.__chunks = b''
122
122
  # try to fetch a package
123
123
  payload, remaining = WebSocket.parse(stream=data)
124
- tail_len = len(remaining)
125
- if tail_len > 0:
124
+ self.__package_received = payload is not None
125
+ remain_len = len(remaining)
126
+ if remain_len > 0:
126
127
  # put the remaining data back to memory cache
127
128
  self.__chunks = remaining + self.__chunks
128
129
  pack = None
129
130
  if payload is not None:
130
131
  data_len = len(data)
131
- pack_len = data_len - tail_len
132
+ pack_len = data_len - len(remaining)
132
133
  if pack_len > 0:
133
134
  pack = data[:pack_len]
134
- return pack, payload
135
+ return pack, payload, remain_len
135
136
 
136
137
  # Override
137
138
  def process_received(self, data: bytes):
@@ -144,7 +145,7 @@ class WSDocker(PlainDocker, DeparturePacker):
144
145
  data = b''
145
146
 
146
147
  # Override
147
- def _get_arrival(self, data: bytes) -> Optional[Arrival]:
148
+ def _get_arrivals(self, data: bytes) -> List[Arrival]:
148
149
  # check for first request
149
150
  if self.__handshaking:
150
151
  # join the data to the memory cache
@@ -159,11 +160,22 @@ class WSDocker(PlainDocker, DeparturePacker):
159
160
  elif len(data) < self.MAX_PACK_LENGTH:
160
161
  # waiting for more data
161
162
  self.__chunks = data + self.__chunks
162
- else:
163
- # normal state
164
- pack, payload = self._parse_package(data=data)
165
- if pack is not None:
166
- return WSArrival(package=pack, payload=payload)
163
+ return []
164
+ # normal state
165
+ ships = []
166
+ while True:
167
+ pack, payload, remain_len = self._parse_package(data=data)
168
+ if pack is None:
169
+ # waiting for more data
170
+ break
171
+ ships.append(WSArrival(package=pack, payload=payload))
172
+ if remain_len > 0:
173
+ # continue to check the tail
174
+ data = b''
175
+ else:
176
+ # all data processed
177
+ break
178
+ return ships
167
179
 
168
180
  # Override
169
181
  def _check_arrival(self, ship: Arrival) -> Optional[Arrival]:
@@ -32,8 +32,6 @@
32
32
  from .session import ServerSession
33
33
  from .session_center import SessionCenter # SessionPool
34
34
 
35
- from .broadcast import BroadcastRecipientManager
36
-
37
35
  from .push import BadgeKeeper, PushService
38
36
  from .push import PushCenter
39
37
 
@@ -56,8 +54,6 @@ __all__ = [
56
54
  # Session
57
55
  'ServerSession', 'SessionCenter', # 'SessionPool',
58
56
 
59
- 'BroadcastRecipientManager',
60
-
61
57
  # Push Notification
62
58
  'BadgeKeeper', 'PushService',
63
59
  'PushCenter',
@@ -28,11 +28,13 @@
28
28
  # SOFTWARE.
29
29
  # ==============================================================================
30
30
 
31
+ import threading
31
32
  from abc import ABC
32
- from typing import Dict, List
33
+ from typing import Dict, List, Set
33
34
 
35
+ from dimsdk import DateTime
34
36
  from dimsdk import FrequencyChecker
35
- from dimsdk import ID, Document
37
+ from dimsdk import EntityType, ID, Document
36
38
  from dimsdk import Envelope, InstantMessage
37
39
  from dimsdk import Command, MetaCommand, DocumentCommand
38
40
  from dimsdk import Station
@@ -40,8 +42,10 @@ from dimsdk import Station
40
42
  from ..common import AccountDBI
41
43
  from ..common import CommonArchivist
42
44
  from ..common import CommonFacebook, CommonMessenger
45
+ from ..common import StationInfo
43
46
 
44
- from .broadcast import broadcast_reliable_message
47
+ from .dispatcher import Dispatcher
48
+ from .session_center import SessionCenter
45
49
 
46
50
 
47
51
  def get_facebook(archivist: CommonArchivist):
@@ -67,21 +71,79 @@ class ServerArchivist(CommonArchivist, ABC):
67
71
  super().__init__(database=database)
68
72
  self.__document_responses = FrequencyChecker(expires=self.RESPOND_EXPIRES)
69
73
  self.__last_active_members: Dict[ID, ID] = {} # group => member
70
-
71
- def _broadcast_command(self, content: Command, receiver: ID) -> bool:
74
+ # neighbor stations
75
+ self.__neighbors = set()
76
+ self.__lock = threading.Lock()
77
+ self.__expires = 0
78
+
79
+ @property
80
+ def active_stations(self) -> Set[ID]:
81
+ """ get neighbor stations connected to current station """
82
+ now = DateTime.now()
83
+ with self.__lock:
84
+ if self.__expires < now.timestamp:
85
+ neighbors = set()
86
+ center = SessionCenter()
87
+ all_users = center.all_users()
88
+ for item in all_users:
89
+ if item.type == EntityType.STATION:
90
+ neighbors.add(item)
91
+ self.__neighbors = neighbors
92
+ self.__expires = now.timestamp + 128
93
+ return self.__neighbors
94
+
95
+ @property
96
+ def all_stations(self) -> List[StationInfo]:
97
+ """ get stations from database """
98
+ dispatcher = Dispatcher()
99
+ db = dispatcher.sdb
100
+ # TODO: get chosen provider
101
+ providers = db.all_providers()
102
+ assert len(providers) > 0, 'service provider not found'
103
+ gsp = providers[0].identifier
104
+ return db.all_stations(provider=gsp)
105
+
106
+ @property
107
+ def all_neighbors(self) -> Set[ID]:
108
+ """ get all stations """
109
+ neighbors = set()
110
+ # get stations from chosen provider
111
+ chosen_stations = self.all_stations
112
+ for item in chosen_stations:
113
+ sid = item.identifier
114
+ if sid is None or sid.is_broadcast:
115
+ continue
116
+ neighbors.add(sid)
117
+ # get neighbor station from session server
118
+ proactive_neighbors = self.active_stations
119
+ for sid in proactive_neighbors:
120
+ if sid is None or sid.is_broadcast:
121
+ self.error(msg='neighbor station ID error: %s' % sid)
122
+ continue
123
+ neighbors.add(sid)
124
+ return neighbors
125
+
126
+ def _broadcast_command(self, command: Command) -> bool:
72
127
  facebook = get_facebook(archivist=self)
73
128
  messenger = get_messenger(archivist=self)
74
129
  if facebook is None or messenger is None:
75
130
  self.error(msg='twins not ready yet: %s, %s' % (facebook, messenger))
76
131
  return False
77
132
  sid = facebook.current_user.identifier
78
- env = Envelope.create(sender=sid, receiver=receiver)
79
- i_msg = InstantMessage.create(head=env, body=content)
133
+ env = Envelope.create(sender=sid, receiver=Station.EVERY)
134
+ i_msg = InstantMessage.create(head=env, body=command)
80
135
  # pack & deliver message
81
136
  s_msg = messenger.encrypt_message(msg=i_msg)
82
137
  r_msg = messenger.sign_message(msg=s_msg)
83
- cnt = broadcast_reliable_message(msg=r_msg, station=sid)
84
- return cnt > 0
138
+ # dispatch
139
+ dispatcher = Dispatcher()
140
+ neighbors = self.all_neighbors
141
+ for receiver in neighbors:
142
+ if receiver == sid:
143
+ self.debug(msg='skip cycled message: %s -> %s' % (sid, receiver))
144
+ continue
145
+ dispatcher.deliver_message(msg=r_msg, receiver=receiver)
146
+ return len(neighbors) > 0
85
147
 
86
148
  # protected
87
149
  def is_documents_respond_expired(self, identifier: ID, force: bool) -> bool:
@@ -98,7 +160,7 @@ class ServerArchivist(CommonArchivist, ABC):
98
160
  return False
99
161
  self.info(msg='querying meta for: %s' % identifier)
100
162
  command = MetaCommand.query(identifier=identifier)
101
- return self._broadcast_command(content=command, receiver=Station.EVERY)
163
+ return self._broadcast_command(command=command)
102
164
 
103
165
  # Override
104
166
  def query_documents(self, identifier: ID, documents: List[Document]) -> bool:
@@ -109,7 +171,7 @@ class ServerArchivist(CommonArchivist, ABC):
109
171
  last_time = self.get_last_document_time(identifier=identifier, documents=documents)
110
172
  self.info(msg='querying document for: %s, last time: %s' % (identifier, last_time))
111
173
  command = DocumentCommand.query(identifier=identifier, last_time=last_time)
112
- return self._broadcast_command(content=command, receiver=Station.EVERY)
174
+ return self._broadcast_command(command=command)
113
175
 
114
176
  # Override
115
177
  def query_members(self, group: ID, members: List[ID]) -> bool:
@@ -31,7 +31,7 @@
31
31
  import time
32
32
  from typing import Optional, Union, List, Dict
33
33
 
34
- from dimsdk import EntityType, ID, ANYONE
34
+ from dimsdk import EntityType, ID, ANYONE, EVERYONE
35
35
  from dimsdk import Station
36
36
  from dimsdk import InstantMessage, ReliableMessage
37
37
  from dimsdk import Envelope
@@ -58,7 +58,7 @@ from .cpu import DocumentCommandProcessor
58
58
 
59
59
  from .packer import FilterManager
60
60
  from .dispatcher import Dispatcher
61
- from .broadcast import broadcast_reliable_message
61
+ from .archivist import ServerArchivist
62
62
 
63
63
 
64
64
  class ServerMessageProcessor(MessageProcessor, Logging):
@@ -95,8 +95,7 @@ class ServerMessageProcessor(MessageProcessor, Logging):
95
95
  messenger = self.messenger
96
96
  session = messenger.session
97
97
  current = self.facebook.current_user
98
- sid = current.identifier
99
- sender = msg.sender
98
+ station = current.identifier
100
99
  receiver = msg.receiver
101
100
  # 0. verify message
102
101
  s_msg = messenger.verify_message(msg=msg)
@@ -104,7 +103,7 @@ class ServerMessageProcessor(MessageProcessor, Logging):
104
103
  # TODO: suspend and waiting for sender's meta if not exists
105
104
  return []
106
105
  # 1. check receiver
107
- if receiver == sid:
106
+ if receiver == station:
108
107
  # message to this station
109
108
  # maybe a meta command, document command, etc ...
110
109
  pass
@@ -131,16 +130,22 @@ class ServerMessageProcessor(MessageProcessor, Logging):
131
130
  # we can trust this message an no need to verify it;
132
131
  # else if sender is a neighbor station,
133
132
  # we can trust it too;
134
- if receiver.is_broadcast:
135
- # broadcast message (to neighbor stations, or station bots)
136
- # e.g.: 'stations@everywhere',
137
- # 'archivist@anywhere', 'announcer@anywhere', 'monitor@anywhere'
138
- broadcast_reliable_message(msg=msg, station=sid)
139
- # if receiver.is_group:
140
- # broadcast message to multiple destinations,
133
+ if receiver == Station.EVERY or receiver == EVERYONE:
134
+ # broadcast message (to neighbor stations)
135
+ # e.g.: 'stations@everywhere'
136
+ self._broadcast_message(msg=msg, station=station)
137
+ # if receiver == 'everyone@everywhere':
138
+ # broadcast message to all destinations,
141
139
  # current station is it's receiver too.
140
+ elif receiver.is_broadcast:
141
+ # broadcast message (to station bots)
142
+ # e.g.: 'archivist@anywhere', 'announcer@anywhere', 'monitor@anywhere'
143
+ self._broadcast_message(msg=msg, station=station)
144
+ return []
142
145
  elif receiver.is_group:
143
- self.error(msg='group message should not send to station: %s -> %s' % (sender, receiver))
146
+ # encrypted group messages should be sent to the group assistant,
147
+ # the station will never process these messages.
148
+ self._split_group_message(msg=msg, station=station)
144
149
  return []
145
150
  else:
146
151
  # this message is not for current station,
@@ -180,6 +185,32 @@ class ServerMessageProcessor(MessageProcessor, Logging):
180
185
  else:
181
186
  return [r_msg]
182
187
 
188
+ def _broadcast_message(self, msg: ReliableMessage, station: ID):
189
+ """ broadcast message to neighbor stations """
190
+ archivist = self.facebook.archivist
191
+ assert isinstance(archivist, ServerArchivist)
192
+ neighbors = archivist.all_neighbors
193
+ sender = msg.sender
194
+ self.info(msg='broadcast message %s -> %s (%s): from station %s to %s'
195
+ % (sender, msg.receiver, msg.group, station, neighbors))
196
+ # dispatch
197
+ dispatcher = Dispatcher()
198
+ for receiver in neighbors:
199
+ if receiver == sender:
200
+ self.debug(msg='skip cycled message: %s -> %s' % (sender, receiver))
201
+ continue
202
+ elif receiver == station:
203
+ self.debug(msg='skip current station: %s -> %s' % (sender, receiver))
204
+ continue
205
+ dispatcher.deliver_message(msg=msg, receiver=receiver)
206
+ return len(neighbors) > 0
207
+
208
+ def _split_group_message(self, msg: ReliableMessage, station: ID):
209
+ """ redirect group message to assistant """
210
+ sender = msg.sender
211
+ receiver = msg.receiver
212
+ self.error(msg='group message should not send to station: %s, %s -> %s' % (station, sender, receiver))
213
+
183
214
  def _deliver_message(self, msg: ReliableMessage) -> List[ReliableMessage]:
184
215
  messenger = self.messenger
185
216
  current = self.facebook.current_user