holado 0.2.8__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of holado might be problematic. Click here for more details.

Files changed (110) hide show
  1. holado/common/handlers/undefined.py +7 -1
  2. {holado-0.2.8.dist-info → holado-0.4.0.dist-info}/METADATA +4 -1
  3. {holado-0.2.8.dist-info → holado-0.4.0.dist-info}/RECORD +108 -101
  4. holado_ais/ais/ais_messages.py +97 -6
  5. holado_ais/tests/behave/steps/ais/ais_manager_steps.py +1 -1
  6. holado_ais/tests/behave/steps/ais/ais_messages_steps.py +45 -6
  7. holado_binary/ipc/bit_series.py +3 -3
  8. holado_binary/tests/behave/steps/ipc/binary_steps.py +1 -1
  9. holado_binary/tests/behave/steps/ipc/bit_series_steps.py +4 -3
  10. holado_context/tests/behave/steps/private/common/context_steps.py +1 -1
  11. holado_core/common/resource/persisted_data_manager.py +13 -16
  12. holado_core/common/resource/resource_manager.py +10 -10
  13. holado_core/common/tables/converters/table_converter.py +47 -9
  14. holado_core/common/tables/table_manager.py +6 -7
  15. holado_core/common/tables/table_with_header.py +6 -0
  16. holado_core/tests/behave/steps/common/common_steps.py +2 -1
  17. holado_core/tests/behave/steps/common/config_steps.py +1 -1
  18. holado_core/tests/behave/steps/common/resource_steps.py +1 -1
  19. holado_core/tests/behave/steps/common/tables_steps.py +18 -2
  20. holado_data/data/generator/generator_manager.py +39 -0
  21. holado_data/tests/behave/steps/data/generator_steps.py +1 -1
  22. holado_data/tests/behave/steps/tools/utils_steps.py +1 -2
  23. holado_db/tests/behave/steps/tools/db/db_client_steps.py +1 -1
  24. holado_db/tests/behave/steps/tools/db/postgresql_client_steps.py +1 -1
  25. holado_db/tests/behave/steps/tools/db/sqlite_client_steps.py +1 -1
  26. holado_db/tools/db/clients/base/db_client.py +81 -28
  27. holado_db/tools/db/clients/postgresql/postgresql_client.py +17 -7
  28. holado_db/tools/db/query/base/query_builder.py +58 -7
  29. holado_db/tools/db/query/pypika/pypika_query_builder.py +73 -21
  30. holado_docker/tests/behave/steps/tools/docker_steps.py +1 -1
  31. holado_grpc/tests/behave/steps/api/grpc_client_steps.py +1 -1
  32. holado_grpc/tests/behave/steps/private/api/grpc_steps.py +1 -1
  33. holado_json/tests/behave/steps/ipc/json_steps.py +1 -1
  34. holado_keycloak/tests/behave/steps/tools/keycloak_client_steps.py +1 -1
  35. holado_multitask/tests/behave/steps/multiprocessing_steps.py +1 -1
  36. holado_multitask/tests/behave/steps/multithreading_steps.py +1 -1
  37. holado_protobuf/ipc/protobuf/types/google/protobuf.py +1 -1
  38. holado_protobuf/tests/behave/steps/ipc/protobuf_steps.py +1 -1
  39. holado_python/common/tools/datetime.py +31 -12
  40. holado_python/standard_library/socket/blocking_socket.py +37 -24
  41. holado_python/standard_library/socket/message_socket.py +11 -3
  42. holado_python/standard_library/socket/non_blocking_socket.py +24 -24
  43. holado_python/standard_library/socket/socket.py +132 -19
  44. holado_python/standard_library/ssl/resources/certificates/NOTES.txt +1 -1
  45. holado_python/standard_library/ssl/resources/certificates/rootCACert.pem +24 -0
  46. holado_python/standard_library/ssl/resources/certificates/tcpbin.crt +21 -0
  47. holado_python/standard_library/ssl/resources/certificates/tcpbin.key +28 -0
  48. holado_python/standard_library/ssl/ssl.py +138 -21
  49. holado_python/tests/behave/steps/convert_steps.py +1 -1
  50. holado_python/tests/behave/steps/iterable_steps.py +1 -1
  51. holado_python/tests/behave/steps/standard_library/csv_steps.py +1 -1
  52. holado_python/tests/behave/steps/standard_library/datetime_steps.py +1 -1
  53. holado_python/tests/behave/steps/standard_library/hashlib_steps.py +2 -2
  54. holado_python/tests/behave/steps/standard_library/multiprocessing_steps.py +1 -1
  55. holado_python/tests/behave/steps/standard_library/queue_steps.py +1 -1
  56. holado_python/tests/behave/steps/standard_library/socket_steps.py +132 -18
  57. holado_python/tests/behave/steps/standard_library/ssl_steps.py +87 -16
  58. holado_rabbitmq/tests/behave/steps/tools/rabbitmq_client_steps.py +48 -20
  59. holado_rabbitmq/tests/behave/steps/tools/rabbitmq_server_steps.py +1 -1
  60. holado_rabbitmq/tools/rabbitmq/rabbitmq_client.py +19 -13
  61. holado_rabbitmq/tools/rabbitmq/rabbitmq_manager.py +2 -29
  62. holado_redis/tests/behave/steps/tools/redis_client_steps.py +1 -1
  63. holado_rest/api/rest/rest_client.py +18 -1
  64. holado_rest/api/rest/rest_manager.py +5 -0
  65. holado_rest/tests/behave/steps/api/rest_client_steps.py +52 -11
  66. holado_rest/tests/behave/steps/private/api/rest_steps.py +1 -1
  67. holado_s3/tests/behave/steps/private/tools/s3_steps.py +1 -1
  68. holado_s3/tests/behave/steps/tools/s3_client_steps.py +1 -1
  69. holado_s3/tests/behave/steps/tools/s3_server_steps.py +1 -1
  70. holado_scripting/tests/behave/steps/common/tools/variable_convert_steps.py +3 -2
  71. holado_scripting/tests/behave/steps/common/tools/variable_new_steps.py +1 -1
  72. holado_scripting/tests/behave/steps/common/tools/variable_steps.py +1 -1
  73. holado_scripting/tests/behave/steps/common/tools/variable_verify_steps.py +1 -1
  74. holado_scripting/tests/behave/steps/scenario/function_steps.py +1 -1
  75. holado_scripting/tests/behave/steps/scenario/if_steps.py +1 -1
  76. holado_scripting/tests/behave/steps/scenario/loop_steps.py +1 -1
  77. holado_sftp/tests/behave/steps/private/tools/sftp_steps.py +1 -1
  78. holado_sftp/tests/behave/steps/tools/sftp_client_steps.py +1 -1
  79. holado_sftp/tests/behave/steps/tools/sftp_server_steps.py +1 -1
  80. holado_swagger/tests/behave/steps/swagger_hub/mockserver_steps.py +1 -1
  81. holado_system/system/command/command.py +14 -9
  82. holado_system/tests/behave/steps/system/commands_steps.py +1 -1
  83. holado_system/tests/behave/steps/system/file_steps.py +1 -1
  84. holado_system/tests/behave/steps/system/system_steps.py +1 -1
  85. holado_test/scenario/step_tools.py +1 -1
  86. holado_test/scenario/tester_tools.py +6 -3
  87. holado_test/tests/behave/steps/scenario/exception_steps.py +1 -1
  88. holado_test/tests/behave/steps/scenario/scenario_steps.py +1 -1
  89. holado_test/tests/behave/steps/scenario/tester_steps.py +4 -4
  90. holado_value/common/tables/converters/value_table_converter.py +52 -8
  91. holado_value/common/tables/value_table_manager.py +0 -10
  92. holado_ws/tests/behave/steps/api/web_service_steps.py +1 -1
  93. holado_yaml/tests/behave/steps/yaml_steps.py +1 -1
  94. holado_yaml/yaml/yaml_manager.py +2 -2
  95. test_holado/features/NonReg/common/tables/table.feature +30 -24
  96. test_holado/features/NonReg/holado_ais/ais_message-bitarray_to_nmea.feature +1 -1
  97. test_holado/features/NonReg/holado_python/standard_library/socket/local_echo_server/socket_reset.feature +191 -0
  98. test_holado/features/NonReg/holado_python/standard_library/{socket_with_ssl.feature → socket/local_echo_server/socket_with_tls_and_verify.feature} +53 -30
  99. test_holado/features/NonReg/holado_python/standard_library/socket/local_echo_server/socket_with_tls_without_verify.feature +299 -0
  100. test_holado/features/NonReg/holado_python/standard_library/{socket.feature → socket/local_echo_server/socket_without_tls.feature} +2 -2
  101. test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_with_mtls.feature +214 -0
  102. test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_with_tls.feature +184 -0
  103. test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_without_tls.feature +169 -0
  104. test_holado/features/NonReg/tools/RabbitMQ.feature +9 -9
  105. test_holado/features/NonReg/tools/RabbitMQ_steps.feature +8 -8
  106. test_holado/logging.conf +5 -3
  107. holado_core/common/transport/__init__.py +0 -0
  108. holado_core/common/transport/crc.py +0 -40
  109. {holado-0.2.8.dist-info → holado-0.4.0.dist-info}/WHEEL +0 -0
  110. {holado-0.2.8.dist-info → holado-0.4.0.dist-info}/licenses/LICENSE +0 -0
@@ -15,7 +15,7 @@
15
15
 
16
16
  from holado_test.scenario.step_tools import StepTools
17
17
  from holado.common.context.session_context import SessionContext
18
- from holado_test.behave.behave import *
18
+ from holado_test.behave.behave import * # @UnusedWildImport
19
19
  from holado_grpc.api.rpc.grpc_client import GRpcClient
20
20
  import logging
21
21
  from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
@@ -14,7 +14,7 @@
14
14
 
15
15
 
16
16
  from holado.common.context.session_context import SessionContext
17
- from holado_test.behave.behave import *
17
+ from holado_test.behave.behave import * # @UnusedWildImport
18
18
  import os.path
19
19
  from holado_core.tools.abstracts.blocking_command_service import BlockingCommandService
20
20
  import logging
@@ -15,7 +15,7 @@
15
15
 
16
16
  from holado_test.scenario.step_tools import StepTools
17
17
  from holado.common.context.session_context import SessionContext
18
- from holado_test.behave.behave import *
18
+ from holado_test.behave.behave import * # @UnusedWildImport
19
19
  from holado_json.ipc.json import create_name_value_table_from_json,\
20
20
  create_table_with_header_from_json
21
21
  import logging
@@ -14,7 +14,7 @@
14
14
 
15
15
 
16
16
  from holado.common.context.session_context import SessionContext
17
- from holado_test.behave.behave import *
17
+ from holado_test.behave.behave import * # @UnusedWildImport
18
18
  from holado_keycloak.tools.keycloak.keycloak_client import KeycloakClient
19
19
  import logging
20
20
  from holado_core.common.exceptions.functional_exception import FunctionalException
@@ -15,7 +15,7 @@
15
15
 
16
16
  from holado.common.context.session_context import SessionContext
17
17
  from holado_core.common.exceptions.functional_exception import FunctionalException
18
- from holado_test.behave.behave import *
18
+ from holado_test.behave.behave import * # @UnusedWildImport
19
19
  from holado_core.common.block.function import Function
20
20
  from holado_test.scenario.step_tools import StepTools
21
21
  import logging
@@ -21,7 +21,7 @@ from holado_test.scenario.step_tools import StepTools
21
21
  import logging
22
22
  from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
23
23
  from holado_scripting.common.tools.evaluate_parameters import EvaluateParameters
24
- from holado_test.behave.behave import *
24
+ from holado_test.behave.behave import * # @UnusedWildImport
25
25
  from holado_multitask.multithreading.periodicfunctionthreaded import PeriodicFunctionThreaded
26
26
  from holado.common.handlers.undefined import undefined_argument
27
27
 
@@ -41,7 +41,7 @@ class Duration(Type):
41
41
  elif isinstance(value, timedelta):
42
42
  obj.FromTimedelta(value)
43
43
  elif isinstance(value, int) or isinstance(value, float):
44
- nanos = value * 1e9
44
+ nanos = round(value * 1e9)
45
45
  obj.FromNanoseconds(nanos)
46
46
  elif isinstance(value, str):
47
47
  if re.match(r"-?\d+(?:.\d+)?s", value):
@@ -15,7 +15,7 @@
15
15
 
16
16
  from holado_test.scenario.step_tools import StepTools
17
17
  from holado.common.context.session_context import SessionContext
18
- from holado_test.behave.behave import *
18
+ from holado_test.behave.behave import * # @UnusedWildImport
19
19
  from holado_core.common.exceptions.functional_exception import FunctionalException
20
20
  from holado_protobuf.ipc.protobuf.protobuf_messages import ProtobufMessages
21
21
  import logging
@@ -26,6 +26,7 @@ from datetime import timedelta
26
26
  from dateutil.relativedelta import relativedelta
27
27
  import math
28
28
  from decimal import Decimal
29
+ from holado_python.standard_library.typing import Typing
29
30
 
30
31
  logger = logging.getLogger(__name__)
31
32
 
@@ -35,6 +36,7 @@ EPOCH_JULIAN_CNES = datetime.datetime(year=1950, month=1, day=1, tzinfo=datetime
35
36
  EPOCH_JULIAN_NASA = datetime.datetime(year=1968, month=5, day=24, tzinfo=datetime.timezone.utc)
36
37
 
37
38
  FORMAT_DATETIME_ISO = '%Y-%m-%dT%H:%M:%S.%fZ'
39
+ FORMAT_DATETIME_ISO_SECONDS = '%Y-%m-%dT%H:%M:%SZ'
38
40
  FORMAT_DATETIME_HUMAN_SECOND = '%Y-%m-%d %H:%M:%S'
39
41
 
40
42
 
@@ -159,12 +161,24 @@ class DateTime(object):
159
161
 
160
162
  ### Conversions to/from timestamp
161
163
 
164
+ @classmethod
165
+ def _get_epoch_datetime(cls, epoch):
166
+ res = None
167
+ if epoch is not None:
168
+ if isinstance(epoch, str):
169
+ if epoch in ['EPOCH_1970', 'EPOCH_JULIAN_CNES', 'EPOCH_JULIAN_NASA']:
170
+ res = eval(epoch)
171
+ else:
172
+ res = DateTime.str_2_datetime(epoch)
173
+ elif isinstance(epoch, datetime.datetime):
174
+ res = epoch
175
+ else:
176
+ raise TechnicalException(f"Epoch can be a datetime, a str containing a datetime, a str containing an EPOCH name, or None")
177
+ return res
178
+
162
179
  @classmethod
163
180
  def datetime_to_timestamp(cls, dt, epoch=None):
164
- if epoch is not None and isinstance(epoch, str):
165
- epoch = DateTime.str_2_datetime(epoch)
166
- elif epoch is not None and not isinstance(epoch, datetime.datetime):
167
- raise TechnicalException(f"Parameter 'epoch' should be a datetime or None")
181
+ epoch = cls._get_epoch_datetime(epoch)
168
182
 
169
183
  if epoch is None:
170
184
  return dt.timestamp()
@@ -176,12 +190,17 @@ class DateTime(object):
176
190
  """
177
191
  Convert timestamp to UTC datetime
178
192
  """
179
- if epoch is not None and isinstance(epoch, str):
180
- epoch = DateTime.str_2_datetime(epoch)
181
- elif epoch is not None and not isinstance(epoch, datetime.datetime):
182
- raise TechnicalException(f"Parameter 'epoch' should be a datetime or None")
193
+ epoch = cls._get_epoch_datetime(epoch)
194
+
195
+ if isinstance(ts, str):
196
+ if Converter.is_float(ts) or Converter.is_integer(ts):
197
+ ts = Converter.to_float(ts)
198
+ else:
199
+ raise FunctionalException(f"Timestamp is a string that doesn't contain a float|int (obtained timestamp: '{ts}')")
200
+ if not isinstance(ts, float) and not isinstance(ts, int):
201
+ raise TechnicalException(f"Timestamp must be a float|int or a string containing a float|int (obtained type: {Typing.get_object_class_fullname(ts)})")
183
202
 
184
- res = datetime.datetime.utcfromtimestamp(ts)
203
+ res = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
185
204
  if epoch is not None:
186
205
  res = epoch + (res - EPOCH_1970)
187
206
  return res
@@ -212,9 +231,9 @@ class DateTime(object):
212
231
  """
213
232
  Convert (seconds, nanos) to timestamp.
214
233
  By default, it returns a timestamp as float with nanos rounded to microsecond precision.
215
- To keep nanosecond precision, set cast_type=None and round_func=None, it will returns a timestamp as decimal.Decimal.
216
- @param cast_func: Type of the returned value (default: float). The type can also be decimal.Decimal, usefull to keep nanosecond precision.
217
- @param round_func: defines the rounding method applied to microseconds (default: round ; possible methods: round, math.ceil, math.trunc)
234
+ To keep nanosecond precision, set cast_func=None and round_func=None, it will returns a timestamp as decimal.Decimal.
235
+ @param cast_func: Type of the returned value (default: float). If None, no cast is applied, and result is of type decimal.Decimal, usefull to keep nanosecond precision.
236
+ @param round_func: defines the rounding method applied to microseconds (default: round ; possible methods: round, math.ceil, math.trunc). If None, value is not rounded.
218
237
  """
219
238
  res_dec = Decimal(seconds) + Decimal(nanos) / Decimal(1e9)
220
239
  if round_func is not None:
@@ -13,7 +13,7 @@
13
13
 
14
14
  import logging
15
15
  import socket
16
- from holado_python.standard_library.socket.socket import SocketClient, SocketServer
16
+ from holado_python.standard_library.socket.socket import SocketClient, SocketServer, Socket
17
17
  import abc
18
18
  from holado_multitask.multithreading.loopfunctionthreaded import LoopFunctionThreaded
19
19
  import time
@@ -94,10 +94,6 @@ class BlockingSocketClient(SocketClient):
94
94
  if not has_activity and self._idle_sleep_delay is not None:
95
95
  time.sleep(self._idle_sleep_delay)
96
96
 
97
- def do_ssl_handshake(self):
98
- if self.is_with_ssl:
99
- self.internal_socket.do_handshake()
100
-
101
97
 
102
98
  class TCPBlockingSocketClient(BlockingSocketClient):
103
99
  """
@@ -108,16 +104,22 @@ class TCPBlockingSocketClient(BlockingSocketClient):
108
104
  super().__init__(name=name, create_ipv4_socket_kwargs=create_ipv4_socket_kwargs, idle_sleep_delay=idle_sleep_delay, do_run_with_recv=do_run_with_recv, do_run_with_send=do_run_with_send)
109
105
 
110
106
  def create_ipv4_socket(self, host, port, **kwargs):
111
- ssl_context, kwargs = self._new_ssl_context_if_required(**kwargs)
112
-
113
- sock = socket.create_connection((host, port), **kwargs)
107
+ socket_kwargs = self._new_ssl_context_if_required(**kwargs)
114
108
 
115
- if ssl_context:
116
- # do_handshake_on_connect = True
117
- do_handshake_on_connect = False
118
- sock = ssl_context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=do_handshake_on_connect)
119
- self._set_internal_socket(sock, is_with_ssl=True, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
109
+ if self.is_with_ssl:
110
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
111
+ # sock = Socket.create_connection((host, port), do_connect=False, **socket_kwargs)
112
+ # sock = Socket.create_connection((host, port), do_connect=True, **socket_kwargs)
113
+
114
+ wrap_socket_kwargs = self._ssl_wrap_socket_kwargs
115
+ # do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', True)
116
+ do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', False)
117
+ sock = self._ssl_context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=do_handshake_on_connect, **wrap_socket_kwargs)
118
+ self._set_internal_socket(sock, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
119
+
120
+ sock.connect((host, port))
120
121
  else:
122
+ sock = socket.create_connection((host, port), **socket_kwargs)
121
123
  self._set_internal_socket(sock)
122
124
 
123
125
 
@@ -152,21 +154,31 @@ class BlockingSocketServer(SocketServer):
152
154
  read_kwargs = read_kwargs if read_kwargs is not None else {}
153
155
  write_kwargs = write_kwargs if write_kwargs is not None else {}
154
156
 
155
- conn, _ = self.accept()
157
+ conn, from_addr = self.accept()
156
158
  if logger.isEnabledFor(logging.DEBUG):
157
- logger.debug(f"[{self.name}] New connection: {conn}")
159
+ logger.debug(f"[{self.name}] New connection: {conn} from {from_addr}")
160
+
161
+ if self.is_with_ssl:
162
+ wrap_socket_kwargs = self._ssl_wrap_socket_kwargs
163
+ do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', False)
164
+ # do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', True)
165
+ ssocket = self._ssl_context.wrap_socket(conn.internal_socket, server_side=True, do_handshake_on_connect=do_handshake_on_connect)
166
+ conn = Socket.create(name=conn.name, internal_socket=ssocket)
167
+
158
168
  with conn:
169
+ if logger.isEnabledFor(logging.DEBUG):
170
+ logger.debug(f"[{self.name} - {from_addr}] Start read & write loop")
159
171
  while True:
160
172
  data = conn.read(read_bufsize, **read_kwargs)
161
173
  if logger.isEnabledFor(logging.DEBUG):
162
- logger.debug(f"[{self.name}] Received: {data}")
174
+ logger.debug(f"[{self.name} - {from_addr}] Received: {data}")
163
175
 
164
176
  if not data:
165
177
  break
166
178
  result = self._process_received_data(data)
167
179
 
168
180
  if logger.isEnabledFor(logging.DEBUG):
169
- logger.debug(f"[{self.name}] Sending: {result}")
181
+ logger.debug(f"[{self.name} - {from_addr}] Sending: {result}")
170
182
  conn.write(result, **write_kwargs)
171
183
 
172
184
 
@@ -180,14 +192,15 @@ class TCPBlockingSocketServer(BlockingSocketServer):
180
192
  super().__init__(name=name, create_ipv4_socket_kwargs=create_ipv4_socket_kwargs, idle_sleep_delay=idle_sleep_delay)
181
193
 
182
194
  def create_ipv4_socket(self, host, port, **kwargs):
183
- ssl_context, kwargs = self._new_ssl_context_if_required(server_side=True, **kwargs)
195
+ socket_kwargs = self._new_ssl_context_if_required(server_side=True, **kwargs)
184
196
 
185
- sock = socket.create_server((host, port), **kwargs)
186
-
187
- if ssl_context:
188
- do_handshake_on_connect = True
189
- sock = ssl_context.wrap_socket(sock, server_side=True, do_handshake_on_connect=do_handshake_on_connect)
190
- self._set_internal_socket(sock, is_with_ssl=True, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
197
+ if self.is_with_ssl:
198
+ # sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
199
+ # sock.bind((host, port))
200
+ # sock.listen()
201
+ sock = socket.create_server((host, port), **socket_kwargs)
202
+ self._set_internal_socket(sock)
191
203
  else:
204
+ sock = socket.create_server((host, port), **socket_kwargs)
192
205
  self._set_internal_socket(sock)
193
206
 
@@ -44,11 +44,13 @@ class MessageSocketClient(object):
44
44
 
45
45
  @property
46
46
  def messages(self):
47
- return self.__messages
47
+ with self.__messages_lock:
48
+ return copy.copy(self.__messages)
48
49
 
49
50
  @property
50
51
  def nb_messages(self):
51
- return len(self.__messages)
52
+ with self.__messages_lock:
53
+ return len(self.__messages)
52
54
 
53
55
  def _extract_messages_from_data(self, data):
54
56
  while True:
@@ -68,6 +70,10 @@ class MessageSocketClient(object):
68
70
  with self.__messages_lock:
69
71
  self.__messages.append(msg)
70
72
 
73
+ def reset_received_messages(self):
74
+ with self.__messages_lock:
75
+ self.__messages.clear()
76
+
71
77
  def read_message(self):
72
78
  res = None
73
79
  with self.__messages_lock:
@@ -115,12 +121,14 @@ class MessageTCPNonBlockingSocketClient(TCPNonBlockingSocketClient, MessageSocke
115
121
 
116
122
  def _service_connection(self, key, mask, *, read_bufsize=1024, read_kwargs=None, write_kwargs=None):
117
123
  # Read from socket & write in socket
118
- super()._service_connection(key, mask, read_bufsize=read_bufsize, read_kwargs=read_kwargs, write_kwargs=write_kwargs)
124
+ res = super()._service_connection(key, mask, read_bufsize=read_bufsize, read_kwargs=read_kwargs, write_kwargs=write_kwargs)
119
125
 
120
126
  # Extract messages from received data
121
127
  if mask & selectors.EVENT_READ:
122
128
  with self._data_lock:
123
129
  self._extract_messages_from_data(self._data)
130
+
131
+ return res
124
132
 
125
133
 
126
134
 
@@ -17,10 +17,9 @@ from holado_python.standard_library.socket.socket import SocketClient
17
17
  import abc
18
18
  from holado_multitask.multithreading.loopfunctionthreaded import LoopFunctionThreaded
19
19
  import selectors
20
- import ssl
21
- import select
22
20
  from holado.common.handlers.undefined import undefined_argument
23
21
  import time
22
+ import ssl
24
23
 
25
24
  logger = logging.getLogger(__name__)
26
25
 
@@ -84,8 +83,10 @@ class NonBlockingSocketClient(SocketClient):
84
83
  flags = 0
85
84
  else:
86
85
  flags = read_kwargs.get('flags', 0)
87
-
88
- recv_data = sock.recv(read_bufsize, flags)
86
+ try:
87
+ recv_data = sock.recv(read_bufsize, flags)
88
+ except ssl.SSLWantReadError:
89
+ recv_data = None
89
90
  if recv_data:
90
91
  has_activity = True
91
92
  with self._data_lock: # data is self._data
@@ -97,7 +98,10 @@ class NonBlockingSocketClient(SocketClient):
97
98
  with self._data_lock: # data is self._data
98
99
  if data.out_bytes:
99
100
  has_activity = True
100
- sent = sock.send(data.out_bytes)
101
+ try:
102
+ sent = sock.send(data.out_bytes)
103
+ except ssl.SSLWantWriteError:
104
+ sent = 0
101
105
  if sent > 0:
102
106
  data.out_bytes = data.out_bytes[sent:]
103
107
  if logger.isEnabledFor(logging.DEBUG):
@@ -105,17 +109,6 @@ class NonBlockingSocketClient(SocketClient):
105
109
 
106
110
  return has_activity
107
111
 
108
- def do_ssl_handshake(self):
109
- if self.is_with_ssl:
110
- while True:
111
- try:
112
- self.internal_socket.do_handshake()
113
- break
114
- except ssl.SSLWantReadError:
115
- select.select([self.internal_socket], [], [])
116
- except ssl.SSLWantWriteError:
117
- select.select([], [self.internal_socket], [])
118
-
119
112
 
120
113
  class TCPNonBlockingSocketClient(NonBlockingSocketClient):
121
114
  """
@@ -126,16 +119,23 @@ class TCPNonBlockingSocketClient(NonBlockingSocketClient):
126
119
  super().__init__(name=name, create_ipv4_socket_kwargs=create_ipv4_socket_kwargs, idle_sleep_delay=idle_sleep_delay, do_run_with_recv=do_run_with_recv, do_run_with_send=do_run_with_send)
127
120
 
128
121
  def create_ipv4_socket(self, host, port, **kwargs):
129
- ssl_context, kwargs = self._new_ssl_context_if_required(**kwargs)
122
+ socket_kwargs = self._new_ssl_context_if_required(**kwargs)
130
123
 
131
- sock = socket.create_connection((host, port), **kwargs)
132
- sock.setblocking(False)
133
-
134
- if ssl_context:
135
- do_handshake_on_connect = False
136
- sock = ssl_context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=do_handshake_on_connect)
137
- self._set_internal_socket(sock, is_with_ssl=True, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
124
+ if self.is_with_ssl:
125
+ # sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
126
+ # sock = Socket.create_connection((host, port), do_connect=False, **socket_kwargs)
127
+ sock = socket.create_connection((host, port), **socket_kwargs)
128
+ sock.setblocking(False)
129
+
130
+ wrap_socket_kwargs = self._ssl_wrap_socket_kwargs
131
+ do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', False)
132
+ sock = self._ssl_context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=do_handshake_on_connect, **wrap_socket_kwargs)
133
+ self._set_internal_socket(sock, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
134
+
135
+ # sock.connect((host, port))
138
136
  else:
137
+ sock = socket.create_connection((host, port), **socket_kwargs)
138
+ sock.setblocking(False)
139
139
  self._set_internal_socket(sock)
140
140
 
141
141
  # Register socket
@@ -13,7 +13,7 @@
13
13
 
14
14
  import logging
15
15
  import socket
16
- from _socket import SHUT_RDWR
16
+ from _socket import SHUT_RDWR, SOCK_STREAM, error
17
17
  from holado.common.handlers.object import DeleteableObject
18
18
  import abc
19
19
  from holado_core.common.exceptions.functional_exception import FunctionalException
@@ -23,6 +23,10 @@ import threading
23
23
  import types
24
24
  from abc import abstractmethod
25
25
  from holado.common.handlers.undefined import undefined_argument
26
+ import copy
27
+ from holado_python.standard_library.typing import Typing
28
+ import ssl
29
+ import select
26
30
 
27
31
  logger = logging.getLogger(__name__)
28
32
 
@@ -35,6 +39,62 @@ class Socket(DeleteableObject):
35
39
  """
36
40
  __metaclass__ = abc.ABCMeta
37
41
 
42
+ @classmethod
43
+ def create(cls, name, internal_socket):
44
+ res = cls.__new__(cls, name=name)
45
+ res.__init__(name=name)
46
+ res._set_internal_socket(internal_socket)
47
+ return res
48
+
49
+ @classmethod
50
+ def create_connection(cls, address, timeout=undefined_argument,
51
+ source_address=None, do_connect=True):
52
+ """Connect to *address* and return the socket object.
53
+
54
+ Convenience function. Connect to *address* (a 2-tuple ``(host,
55
+ port)``) and return the socket object. Passing the optional
56
+ *timeout* parameter will set the timeout on the socket instance
57
+ before attempting to connect. If no *timeout* is supplied, the
58
+ global default timeout setting returned by :func:`getdefaulttimeout`
59
+ is used. If *source_address* is set it must be a tuple of (host, port)
60
+ for the socket to bind as a source address before making the connection.
61
+ A host of '' or port 0 tells the OS to use the default.
62
+
63
+ Note: Implementation is a copy of socket.create_connection, but allowing to not connect.
64
+ If do_connect==True, implementation is identical to socket.create_connection.
65
+ """
66
+
67
+ host, port = address
68
+ err = None
69
+ for res in socket.getaddrinfo(host, port, 0, SOCK_STREAM):
70
+ af, socktype, proto, canonname, sa = res # @UnusedVariable
71
+ sock = None
72
+ try:
73
+ sock = socket.socket(af, socktype, proto)
74
+ if timeout is not undefined_argument:
75
+ sock.settimeout(timeout)
76
+ if source_address:
77
+ sock.bind(source_address)
78
+ if do_connect:
79
+ sock.connect(sa)
80
+ # Break explicitly a reference cycle
81
+ err = None
82
+ return sock
83
+
84
+ except error as _:
85
+ err = _
86
+ if sock is not None:
87
+ sock.close()
88
+
89
+ if err is not None:
90
+ try:
91
+ raise err
92
+ finally:
93
+ # Break explicitly a reference cycle
94
+ err = None
95
+ else:
96
+ raise error("getaddrinfo returns an empty list")
97
+
38
98
  def __init__(self, *, name=None, create_ipv4_socket_kwargs=None, idle_sleep_delay=undefined_argument):
39
99
  """Socket constructor
40
100
  @param name: Socket name
@@ -44,7 +104,7 @@ class Socket(DeleteableObject):
44
104
  if name is None and create_ipv4_socket_kwargs is not None:
45
105
  if not set(create_ipv4_socket_kwargs.keys()).issuperset({'host', 'port'}):
46
106
  raise FunctionalException(f"Parameters 'host' and 'port' must be defined")
47
- name = f"{create_ipv4_socket_kwargs['host']}:{create_ipv4_socket_kwargs['port']}"
107
+ name = f"{Typing.get_object_class_name(self)}({create_ipv4_socket_kwargs['host']}:{create_ipv4_socket_kwargs['port']})"
48
108
 
49
109
  super().__init__(name)
50
110
  self.__socket = None
@@ -53,7 +113,8 @@ class Socket(DeleteableObject):
53
113
  self.__idle_sleep_delay = idle_sleep_delay if idle_sleep_delay is not undefined_argument else 0.01
54
114
 
55
115
  # SSL management
56
- self.__is_with_ssl = False
116
+ self.__ssl_kwargs_tuple = None
117
+ self.__ssl_context = None
57
118
  self.__is_ssl_handshake_done = False
58
119
 
59
120
  if create_ipv4_socket_kwargs is not None:
@@ -78,15 +139,30 @@ class Socket(DeleteableObject):
78
139
 
79
140
  @property
80
141
  def is_with_ssl(self):
81
- return self.__is_with_ssl
142
+ return self.__ssl_context is not None
143
+
144
+ @property
145
+ def _ssl_kwargs(self):
146
+ return copy.copy(self.__ssl_kwargs_tuple[0]) if self.__ssl_kwargs_tuple and self.__ssl_kwargs_tuple[0] is not None else None
147
+
148
+ @property
149
+ def _ssl_context_kwargs(self):
150
+ return copy.copy(self.__ssl_kwargs_tuple[1]) if self.__ssl_kwargs_tuple and self.__ssl_kwargs_tuple[1] is not None else None
151
+
152
+ @property
153
+ def _ssl_wrap_socket_kwargs(self):
154
+ return copy.copy(self.__ssl_kwargs_tuple[2]) if self.__ssl_kwargs_tuple and self.__ssl_kwargs_tuple[2] is not None else None
155
+
156
+ @property
157
+ def _ssl_context(self):
158
+ return self.__ssl_context
82
159
 
83
160
  @property
84
161
  def _idle_sleep_delay(self):
85
162
  return self.__idle_sleep_delay
86
163
 
87
- def _set_internal_socket(self, socket, is_with_ssl=False, is_ssl_handshake_done_on_connect=True):
164
+ def _set_internal_socket(self, socket, is_ssl_handshake_done_on_connect=True):
88
165
  self.__socket = socket
89
- self.__is_with_ssl = is_with_ssl
90
166
  self.__is_ssl_handshake_done = is_ssl_handshake_done_on_connect
91
167
 
92
168
  @abc.abstractmethod
@@ -98,7 +174,9 @@ class Socket(DeleteableObject):
98
174
  try:
99
175
  self.__socket.shutdown(SHUT_RDWR)
100
176
  except OSError as exc:
101
- if 'Errno 107' in str(exc):
177
+ if 'Errno 9' in str(exc):
178
+ logger.info(f"Got following error on socket shutdown (known to appear when connection is not open correctly or already closed): {exc}")
179
+ elif 'Errno 107' in str(exc):
102
180
  logger.info(f"Got following error on socket shutdown (known to appear sometimes during shutdown): {exc}")
103
181
  else:
104
182
  raise exc
@@ -149,19 +227,45 @@ class Socket(DeleteableObject):
149
227
  logger.debug(f"[{self.name}] Sent {res}/{len(data_bytes)} bytes of [{data_bytes}] (type: {type(data_bytes)})")
150
228
  return res
151
229
 
152
- def _new_ssl_context_if_required(self, server_side=False, **socket_kwargs):
153
- """Return a SSLContext if required, and the remaining socket kwargs.
230
+ def _extract_ssl_kwargs(self, socket_kwargs):
231
+ """Pop from socket kwargs: global ssl kwargs, SSLContext kwargs, and wrap_socket kwargs.
154
232
  """
155
- res = None
233
+ ssl_kwargs, context_kwargs, wrap_socket_kwargs = None, {}, {}
156
234
  if Tools.has_sub_kwargs(socket_kwargs, 'ssl.'):
157
235
  ssl_kwargs = Tools.pop_sub_kwargs(socket_kwargs, 'ssl.')
158
- res = SslManager.new_ssl_context(server_side=server_side, flat_kwargs=ssl_kwargs)
159
- return res, socket_kwargs
236
+ if Tools.has_sub_kwargs(ssl_kwargs, 'context.'):
237
+ context_kwargs = Tools.pop_sub_kwargs(ssl_kwargs, 'context.')
238
+ if Tools.has_sub_kwargs(ssl_kwargs, 'wrap_socket.'):
239
+ wrap_socket_kwargs = Tools.pop_sub_kwargs(ssl_kwargs, 'wrap_socket.')
240
+ self.__ssl_kwargs_tuple = (ssl_kwargs, context_kwargs, wrap_socket_kwargs)
241
+ return socket_kwargs
242
+
243
+ def _new_ssl_context_if_required(self, server_side=False, **kwargs):
244
+ """Return the remaining socket kwargs after ssl kwargs extraction.
245
+ """
246
+ res = self._extract_ssl_kwargs(kwargs)
247
+
248
+ if self._ssl_kwargs is not None:
249
+ self.__ssl_context = SslManager.new_ssl_context(server_side=server_side, ssl_kwargs=self._ssl_kwargs, context_kwargs=self._ssl_context_kwargs)
250
+
251
+ return res
160
252
 
161
253
  def do_ssl_handshake(self):
162
- # Handshake management is performed differently with blocking or non-blocking connection
163
- # See BlockingSocketClient and NonBlockingSocketClient for implementation
164
- raise NotImplementedError()
254
+ if self.is_with_ssl:
255
+ if self.internal_socket.getblocking():
256
+ self.internal_socket.do_handshake()
257
+ # self.internal_socket.do_handshake(block=True)
258
+ else:
259
+ while True:
260
+ try:
261
+ self.internal_socket.do_handshake()
262
+ break
263
+ except ssl.SSLWantReadError:
264
+ select.select([self.internal_socket], [], [])
265
+ except ssl.SSLWantWriteError:
266
+ select.select([], [self.internal_socket], [])
267
+
268
+
165
269
 
166
270
  def ensure_ssl_handshake_is_done(self):
167
271
  if self.is_with_ssl and not self.__is_ssl_handshake_done:
@@ -186,8 +290,8 @@ class SocketClient(Socket):
186
290
  # Data used in start processing
187
291
  self.__data_lock = threading.Lock()
188
292
  self.__data = types.SimpleNamespace(
189
- in_bytes=b"",
190
- out_bytes=b"",
293
+ in_bytes = b'',
294
+ out_bytes = b'',
191
295
  )
192
296
  self.__with_recv = do_run_with_recv
193
297
  self.__with_send = do_run_with_send
@@ -222,12 +326,22 @@ class SocketClient(Socket):
222
326
  # Start thread
223
327
  super()._start_thread(thread)
224
328
 
329
+ @property
330
+ def received_data(self):
331
+ with self.__data_lock:
332
+ res = copy.copy(self.__data.in_bytes)
333
+ return res
334
+
225
335
  @property
226
336
  def received_data_size(self):
227
337
  with self.__data_lock:
228
338
  res = len(self.__data.in_bytes)
229
339
  return res
230
340
 
341
+ def reset_received_data(self):
342
+ with self.__data_lock:
343
+ self.__data.in_bytes = b''
344
+
231
345
  def read(self, bufsize=1024, **kwargs):
232
346
  if self.is_started and self.is_run_with_recv:
233
347
  with self.__data_lock:
@@ -263,8 +377,7 @@ class SocketServer(Socket):
263
377
 
264
378
  def accept(self):
265
379
  conn, addr = self.internal_socket.accept()
266
- res = Socket(name=f"[{self.name}] Connection with {addr}")
267
- res._set_internal_socket(conn)
380
+ res = Socket.create(name=f"[{self.name}] Connection with {addr}", internal_socket=conn)
268
381
  return res, addr
269
382
 
270
383
  @abc.abstractmethod
@@ -1,4 +1,4 @@
1
- localhost.ct and localhost.key were generated with command:
1
+ localhost.crt and localhost.key were generated with command:
2
2
  openssl req -x509 -out localhost.crt -keyout localhost.key \
3
3
  -newkey rsa:2048 -nodes -sha256 \
4
4
  -subj '/CN=localhost' -extensions EXT -config <( \