holado 0.2.8__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of holado might be problematic. Click here for more details.
- holado/common/handlers/undefined.py +7 -1
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/METADATA +4 -1
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/RECORD +106 -99
- holado_ais/ais/ais_messages.py +97 -6
- holado_ais/tests/behave/steps/ais/ais_manager_steps.py +1 -1
- holado_ais/tests/behave/steps/ais/ais_messages_steps.py +45 -6
- holado_binary/ipc/bit_series.py +3 -3
- holado_binary/tests/behave/steps/ipc/binary_steps.py +1 -1
- holado_binary/tests/behave/steps/ipc/bit_series_steps.py +4 -3
- holado_context/tests/behave/steps/private/common/context_steps.py +1 -1
- holado_core/common/resource/persisted_data_manager.py +13 -16
- holado_core/common/resource/resource_manager.py +10 -10
- holado_core/common/tables/converters/table_converter.py +47 -9
- holado_core/common/tables/table_manager.py +6 -7
- holado_core/common/tables/table_with_header.py +6 -0
- holado_core/tests/behave/steps/common/common_steps.py +2 -1
- holado_core/tests/behave/steps/common/config_steps.py +1 -1
- holado_core/tests/behave/steps/common/resource_steps.py +1 -1
- holado_core/tests/behave/steps/common/tables_steps.py +18 -2
- holado_data/data/generator/generator_manager.py +39 -0
- holado_data/tests/behave/steps/data/generator_steps.py +1 -1
- holado_data/tests/behave/steps/tools/utils_steps.py +1 -2
- holado_db/tests/behave/steps/tools/db/db_client_steps.py +1 -1
- holado_db/tests/behave/steps/tools/db/postgresql_client_steps.py +1 -1
- holado_db/tests/behave/steps/tools/db/sqlite_client_steps.py +1 -1
- holado_db/tools/db/clients/base/db_client.py +81 -28
- holado_db/tools/db/clients/postgresql/postgresql_client.py +17 -7
- holado_db/tools/db/query/base/query_builder.py +58 -7
- holado_db/tools/db/query/pypika/pypika_query_builder.py +73 -21
- holado_docker/tests/behave/steps/tools/docker_steps.py +1 -1
- holado_grpc/tests/behave/steps/api/grpc_client_steps.py +1 -1
- holado_grpc/tests/behave/steps/private/api/grpc_steps.py +1 -1
- holado_json/tests/behave/steps/ipc/json_steps.py +1 -1
- holado_keycloak/tests/behave/steps/tools/keycloak_client_steps.py +1 -1
- holado_multitask/tests/behave/steps/multiprocessing_steps.py +1 -1
- holado_multitask/tests/behave/steps/multithreading_steps.py +1 -1
- holado_protobuf/ipc/protobuf/types/google/protobuf.py +1 -1
- holado_protobuf/tests/behave/steps/ipc/protobuf_steps.py +1 -1
- holado_python/common/tools/datetime.py +31 -12
- holado_python/standard_library/socket/blocking_socket.py +37 -24
- holado_python/standard_library/socket/message_socket.py +11 -3
- holado_python/standard_library/socket/non_blocking_socket.py +24 -24
- holado_python/standard_library/socket/socket.py +132 -19
- holado_python/standard_library/ssl/resources/certificates/NOTES.txt +1 -1
- holado_python/standard_library/ssl/resources/certificates/rootCACert.pem +24 -0
- holado_python/standard_library/ssl/resources/certificates/tcpbin.crt +21 -0
- holado_python/standard_library/ssl/resources/certificates/tcpbin.key +28 -0
- holado_python/standard_library/ssl/ssl.py +138 -21
- holado_python/tests/behave/steps/convert_steps.py +1 -1
- holado_python/tests/behave/steps/iterable_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/csv_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/datetime_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/hashlib_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/multiprocessing_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/queue_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/socket_steps.py +132 -18
- holado_python/tests/behave/steps/standard_library/ssl_steps.py +87 -16
- holado_rabbitmq/tests/behave/steps/tools/rabbitmq_client_steps.py +48 -20
- holado_rabbitmq/tests/behave/steps/tools/rabbitmq_server_steps.py +1 -1
- holado_rabbitmq/tools/rabbitmq/rabbitmq_client.py +19 -13
- holado_rabbitmq/tools/rabbitmq/rabbitmq_manager.py +2 -29
- holado_redis/tests/behave/steps/tools/redis_client_steps.py +1 -1
- holado_rest/tests/behave/steps/api/rest_client_steps.py +1 -1
- holado_rest/tests/behave/steps/private/api/rest_steps.py +1 -1
- holado_s3/tests/behave/steps/private/tools/s3_steps.py +1 -1
- holado_s3/tests/behave/steps/tools/s3_client_steps.py +1 -1
- holado_s3/tests/behave/steps/tools/s3_server_steps.py +1 -1
- holado_scripting/tests/behave/steps/common/tools/variable_convert_steps.py +3 -2
- holado_scripting/tests/behave/steps/common/tools/variable_new_steps.py +1 -1
- holado_scripting/tests/behave/steps/common/tools/variable_steps.py +1 -1
- holado_scripting/tests/behave/steps/common/tools/variable_verify_steps.py +1 -1
- holado_scripting/tests/behave/steps/scenario/function_steps.py +1 -1
- holado_scripting/tests/behave/steps/scenario/if_steps.py +1 -1
- holado_scripting/tests/behave/steps/scenario/loop_steps.py +1 -1
- holado_sftp/tests/behave/steps/private/tools/sftp_steps.py +1 -1
- holado_sftp/tests/behave/steps/tools/sftp_client_steps.py +1 -1
- holado_sftp/tests/behave/steps/tools/sftp_server_steps.py +1 -1
- holado_swagger/tests/behave/steps/swagger_hub/mockserver_steps.py +1 -1
- holado_system/system/command/command.py +14 -9
- holado_system/tests/behave/steps/system/commands_steps.py +1 -1
- holado_system/tests/behave/steps/system/file_steps.py +1 -1
- holado_system/tests/behave/steps/system/system_steps.py +1 -1
- holado_test/scenario/step_tools.py +1 -1
- holado_test/scenario/tester_tools.py +6 -3
- holado_test/tests/behave/steps/scenario/exception_steps.py +1 -1
- holado_test/tests/behave/steps/scenario/scenario_steps.py +1 -1
- holado_test/tests/behave/steps/scenario/tester_steps.py +4 -4
- holado_value/common/tables/converters/value_table_converter.py +52 -8
- holado_value/common/tables/value_table_manager.py +0 -10
- holado_ws/tests/behave/steps/api/web_service_steps.py +1 -1
- holado_yaml/tests/behave/steps/yaml_steps.py +1 -1
- holado_yaml/yaml/yaml_manager.py +2 -2
- test_holado/features/NonReg/common/tables/table.feature +30 -24
- test_holado/features/NonReg/holado_ais/ais_message-bitarray_to_nmea.feature +1 -1
- test_holado/features/NonReg/holado_python/standard_library/socket/local_echo_server/socket_reset.feature +191 -0
- test_holado/features/NonReg/holado_python/standard_library/{socket_with_ssl.feature → socket/local_echo_server/socket_with_tls_and_verify.feature} +53 -30
- test_holado/features/NonReg/holado_python/standard_library/socket/local_echo_server/socket_with_tls_without_verify.feature +299 -0
- test_holado/features/NonReg/holado_python/standard_library/{socket.feature → socket/local_echo_server/socket_without_tls.feature} +2 -2
- test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_with_mtls.feature +214 -0
- test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_with_tls.feature +184 -0
- test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_without_tls.feature +169 -0
- test_holado/features/NonReg/tools/RabbitMQ.feature +9 -9
- test_holado/features/NonReg/tools/RabbitMQ_steps.feature +8 -8
- test_holado/logging.conf +5 -3
- holado_core/common/transport/__init__.py +0 -0
- holado_core/common/transport/crc.py +0 -40
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/WHEEL +0 -0
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from holado_test.scenario.step_tools import StepTools
|
|
17
17
|
from holado.common.context.session_context import SessionContext
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_grpc.api.rpc.grpc_client import GRpcClient
|
|
20
20
|
import logging
|
|
21
21
|
from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
from holado.common.context.session_context import SessionContext
|
|
17
|
-
from holado_test.behave.behave import *
|
|
17
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
18
18
|
import os.path
|
|
19
19
|
from holado_core.tools.abstracts.blocking_command_service import BlockingCommandService
|
|
20
20
|
import logging
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from holado_test.scenario.step_tools import StepTools
|
|
17
17
|
from holado.common.context.session_context import SessionContext
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_json.ipc.json import create_name_value_table_from_json,\
|
|
20
20
|
create_table_with_header_from_json
|
|
21
21
|
import logging
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
from holado.common.context.session_context import SessionContext
|
|
17
|
-
from holado_test.behave.behave import *
|
|
17
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
18
18
|
from holado_keycloak.tools.keycloak.keycloak_client import KeycloakClient
|
|
19
19
|
import logging
|
|
20
20
|
from holado_core.common.exceptions.functional_exception import FunctionalException
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from holado.common.context.session_context import SessionContext
|
|
17
17
|
from holado_core.common.exceptions.functional_exception import FunctionalException
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_core.common.block.function import Function
|
|
20
20
|
from holado_test.scenario.step_tools import StepTools
|
|
21
21
|
import logging
|
|
@@ -21,7 +21,7 @@ from holado_test.scenario.step_tools import StepTools
|
|
|
21
21
|
import logging
|
|
22
22
|
from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
|
|
23
23
|
from holado_scripting.common.tools.evaluate_parameters import EvaluateParameters
|
|
24
|
-
from holado_test.behave.behave import *
|
|
24
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
25
25
|
from holado_multitask.multithreading.periodicfunctionthreaded import PeriodicFunctionThreaded
|
|
26
26
|
from holado.common.handlers.undefined import undefined_argument
|
|
27
27
|
|
|
@@ -41,7 +41,7 @@ class Duration(Type):
|
|
|
41
41
|
elif isinstance(value, timedelta):
|
|
42
42
|
obj.FromTimedelta(value)
|
|
43
43
|
elif isinstance(value, int) or isinstance(value, float):
|
|
44
|
-
nanos = value * 1e9
|
|
44
|
+
nanos = round(value * 1e9)
|
|
45
45
|
obj.FromNanoseconds(nanos)
|
|
46
46
|
elif isinstance(value, str):
|
|
47
47
|
if re.match(r"-?\d+(?:.\d+)?s", value):
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from holado_test.scenario.step_tools import StepTools
|
|
17
17
|
from holado.common.context.session_context import SessionContext
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_core.common.exceptions.functional_exception import FunctionalException
|
|
20
20
|
from holado_protobuf.ipc.protobuf.protobuf_messages import ProtobufMessages
|
|
21
21
|
import logging
|
|
@@ -26,6 +26,7 @@ from datetime import timedelta
|
|
|
26
26
|
from dateutil.relativedelta import relativedelta
|
|
27
27
|
import math
|
|
28
28
|
from decimal import Decimal
|
|
29
|
+
from holado_python.standard_library.typing import Typing
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger(__name__)
|
|
31
32
|
|
|
@@ -35,6 +36,7 @@ EPOCH_JULIAN_CNES = datetime.datetime(year=1950, month=1, day=1, tzinfo=datetime
|
|
|
35
36
|
EPOCH_JULIAN_NASA = datetime.datetime(year=1968, month=5, day=24, tzinfo=datetime.timezone.utc)
|
|
36
37
|
|
|
37
38
|
FORMAT_DATETIME_ISO = '%Y-%m-%dT%H:%M:%S.%fZ'
|
|
39
|
+
FORMAT_DATETIME_ISO_SECONDS = '%Y-%m-%dT%H:%M:%SZ'
|
|
38
40
|
FORMAT_DATETIME_HUMAN_SECOND = '%Y-%m-%d %H:%M:%S'
|
|
39
41
|
|
|
40
42
|
|
|
@@ -159,12 +161,24 @@ class DateTime(object):
|
|
|
159
161
|
|
|
160
162
|
### Conversions to/from timestamp
|
|
161
163
|
|
|
164
|
+
@classmethod
|
|
165
|
+
def _get_epoch_datetime(cls, epoch):
|
|
166
|
+
res = None
|
|
167
|
+
if epoch is not None:
|
|
168
|
+
if isinstance(epoch, str):
|
|
169
|
+
if epoch in ['EPOCH_1970', 'EPOCH_JULIAN_CNES', 'EPOCH_JULIAN_NASA']:
|
|
170
|
+
res = eval(epoch)
|
|
171
|
+
else:
|
|
172
|
+
res = DateTime.str_2_datetime(epoch)
|
|
173
|
+
elif isinstance(epoch, datetime.datetime):
|
|
174
|
+
res = epoch
|
|
175
|
+
else:
|
|
176
|
+
raise TechnicalException(f"Epoch can be a datetime, a str containing a datetime, a str containing an EPOCH name, or None")
|
|
177
|
+
return res
|
|
178
|
+
|
|
162
179
|
@classmethod
|
|
163
180
|
def datetime_to_timestamp(cls, dt, epoch=None):
|
|
164
|
-
|
|
165
|
-
epoch = DateTime.str_2_datetime(epoch)
|
|
166
|
-
elif epoch is not None and not isinstance(epoch, datetime.datetime):
|
|
167
|
-
raise TechnicalException(f"Parameter 'epoch' should be a datetime or None")
|
|
181
|
+
epoch = cls._get_epoch_datetime(epoch)
|
|
168
182
|
|
|
169
183
|
if epoch is None:
|
|
170
184
|
return dt.timestamp()
|
|
@@ -176,12 +190,17 @@ class DateTime(object):
|
|
|
176
190
|
"""
|
|
177
191
|
Convert timestamp to UTC datetime
|
|
178
192
|
"""
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
193
|
+
epoch = cls._get_epoch_datetime(epoch)
|
|
194
|
+
|
|
195
|
+
if isinstance(ts, str):
|
|
196
|
+
if Converter.is_float(ts) or Converter.is_integer(ts):
|
|
197
|
+
ts = Converter.to_float(ts)
|
|
198
|
+
else:
|
|
199
|
+
raise FunctionalException(f"Timestamp is a string that doesn't contain a float|int (obtained timestamp: '{ts}')")
|
|
200
|
+
if not isinstance(ts, float) and not isinstance(ts, int):
|
|
201
|
+
raise TechnicalException(f"Timestamp must be a float|int or a string containing a float|int (obtained type: {Typing.get_object_class_fullname(ts)})")
|
|
183
202
|
|
|
184
|
-
res = datetime.datetime.
|
|
203
|
+
res = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
|
|
185
204
|
if epoch is not None:
|
|
186
205
|
res = epoch + (res - EPOCH_1970)
|
|
187
206
|
return res
|
|
@@ -212,9 +231,9 @@ class DateTime(object):
|
|
|
212
231
|
"""
|
|
213
232
|
Convert (seconds, nanos) to timestamp.
|
|
214
233
|
By default, it returns a timestamp as float with nanos rounded to microsecond precision.
|
|
215
|
-
To keep nanosecond precision, set
|
|
216
|
-
@param cast_func: Type of the returned value (default: float).
|
|
217
|
-
@param round_func: defines the rounding method applied to microseconds (default: round ; possible methods: round, math.ceil, math.trunc)
|
|
234
|
+
To keep nanosecond precision, set cast_func=None and round_func=None, it will returns a timestamp as decimal.Decimal.
|
|
235
|
+
@param cast_func: Type of the returned value (default: float). If None, no cast is applied, and result is of type decimal.Decimal, usefull to keep nanosecond precision.
|
|
236
|
+
@param round_func: defines the rounding method applied to microseconds (default: round ; possible methods: round, math.ceil, math.trunc). If None, value is not rounded.
|
|
218
237
|
"""
|
|
219
238
|
res_dec = Decimal(seconds) + Decimal(nanos) / Decimal(1e9)
|
|
220
239
|
if round_func is not None:
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import logging
|
|
15
15
|
import socket
|
|
16
|
-
from holado_python.standard_library.socket.socket import SocketClient, SocketServer
|
|
16
|
+
from holado_python.standard_library.socket.socket import SocketClient, SocketServer, Socket
|
|
17
17
|
import abc
|
|
18
18
|
from holado_multitask.multithreading.loopfunctionthreaded import LoopFunctionThreaded
|
|
19
19
|
import time
|
|
@@ -94,10 +94,6 @@ class BlockingSocketClient(SocketClient):
|
|
|
94
94
|
if not has_activity and self._idle_sleep_delay is not None:
|
|
95
95
|
time.sleep(self._idle_sleep_delay)
|
|
96
96
|
|
|
97
|
-
def do_ssl_handshake(self):
|
|
98
|
-
if self.is_with_ssl:
|
|
99
|
-
self.internal_socket.do_handshake()
|
|
100
|
-
|
|
101
97
|
|
|
102
98
|
class TCPBlockingSocketClient(BlockingSocketClient):
|
|
103
99
|
"""
|
|
@@ -108,16 +104,22 @@ class TCPBlockingSocketClient(BlockingSocketClient):
|
|
|
108
104
|
super().__init__(name=name, create_ipv4_socket_kwargs=create_ipv4_socket_kwargs, idle_sleep_delay=idle_sleep_delay, do_run_with_recv=do_run_with_recv, do_run_with_send=do_run_with_send)
|
|
109
105
|
|
|
110
106
|
def create_ipv4_socket(self, host, port, **kwargs):
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
sock = socket.create_connection((host, port), **kwargs)
|
|
107
|
+
socket_kwargs = self._new_ssl_context_if_required(**kwargs)
|
|
114
108
|
|
|
115
|
-
if
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
sock =
|
|
119
|
-
|
|
109
|
+
if self.is_with_ssl:
|
|
110
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
111
|
+
# sock = Socket.create_connection((host, port), do_connect=False, **socket_kwargs)
|
|
112
|
+
# sock = Socket.create_connection((host, port), do_connect=True, **socket_kwargs)
|
|
113
|
+
|
|
114
|
+
wrap_socket_kwargs = self._ssl_wrap_socket_kwargs
|
|
115
|
+
# do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', True)
|
|
116
|
+
do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', False)
|
|
117
|
+
sock = self._ssl_context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=do_handshake_on_connect, **wrap_socket_kwargs)
|
|
118
|
+
self._set_internal_socket(sock, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
|
|
119
|
+
|
|
120
|
+
sock.connect((host, port))
|
|
120
121
|
else:
|
|
122
|
+
sock = socket.create_connection((host, port), **socket_kwargs)
|
|
121
123
|
self._set_internal_socket(sock)
|
|
122
124
|
|
|
123
125
|
|
|
@@ -152,21 +154,31 @@ class BlockingSocketServer(SocketServer):
|
|
|
152
154
|
read_kwargs = read_kwargs if read_kwargs is not None else {}
|
|
153
155
|
write_kwargs = write_kwargs if write_kwargs is not None else {}
|
|
154
156
|
|
|
155
|
-
conn,
|
|
157
|
+
conn, from_addr = self.accept()
|
|
156
158
|
if logger.isEnabledFor(logging.DEBUG):
|
|
157
|
-
logger.debug(f"[{self.name}] New connection: {conn}")
|
|
159
|
+
logger.debug(f"[{self.name}] New connection: {conn} from {from_addr}")
|
|
160
|
+
|
|
161
|
+
if self.is_with_ssl:
|
|
162
|
+
wrap_socket_kwargs = self._ssl_wrap_socket_kwargs
|
|
163
|
+
do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', False)
|
|
164
|
+
# do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', True)
|
|
165
|
+
ssocket = self._ssl_context.wrap_socket(conn.internal_socket, server_side=True, do_handshake_on_connect=do_handshake_on_connect)
|
|
166
|
+
conn = Socket.create(name=conn.name, internal_socket=ssocket)
|
|
167
|
+
|
|
158
168
|
with conn:
|
|
169
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
170
|
+
logger.debug(f"[{self.name} - {from_addr}] Start read & write loop")
|
|
159
171
|
while True:
|
|
160
172
|
data = conn.read(read_bufsize, **read_kwargs)
|
|
161
173
|
if logger.isEnabledFor(logging.DEBUG):
|
|
162
|
-
logger.debug(f"[{self.name}] Received: {data}")
|
|
174
|
+
logger.debug(f"[{self.name} - {from_addr}] Received: {data}")
|
|
163
175
|
|
|
164
176
|
if not data:
|
|
165
177
|
break
|
|
166
178
|
result = self._process_received_data(data)
|
|
167
179
|
|
|
168
180
|
if logger.isEnabledFor(logging.DEBUG):
|
|
169
|
-
logger.debug(f"[{self.name}] Sending: {result}")
|
|
181
|
+
logger.debug(f"[{self.name} - {from_addr}] Sending: {result}")
|
|
170
182
|
conn.write(result, **write_kwargs)
|
|
171
183
|
|
|
172
184
|
|
|
@@ -180,14 +192,15 @@ class TCPBlockingSocketServer(BlockingSocketServer):
|
|
|
180
192
|
super().__init__(name=name, create_ipv4_socket_kwargs=create_ipv4_socket_kwargs, idle_sleep_delay=idle_sleep_delay)
|
|
181
193
|
|
|
182
194
|
def create_ipv4_socket(self, host, port, **kwargs):
|
|
183
|
-
|
|
195
|
+
socket_kwargs = self._new_ssl_context_if_required(server_side=True, **kwargs)
|
|
184
196
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
sock =
|
|
190
|
-
self._set_internal_socket(sock
|
|
197
|
+
if self.is_with_ssl:
|
|
198
|
+
# sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
199
|
+
# sock.bind((host, port))
|
|
200
|
+
# sock.listen()
|
|
201
|
+
sock = socket.create_server((host, port), **socket_kwargs)
|
|
202
|
+
self._set_internal_socket(sock)
|
|
191
203
|
else:
|
|
204
|
+
sock = socket.create_server((host, port), **socket_kwargs)
|
|
192
205
|
self._set_internal_socket(sock)
|
|
193
206
|
|
|
@@ -44,11 +44,13 @@ class MessageSocketClient(object):
|
|
|
44
44
|
|
|
45
45
|
@property
|
|
46
46
|
def messages(self):
|
|
47
|
-
|
|
47
|
+
with self.__messages_lock:
|
|
48
|
+
return copy.copy(self.__messages)
|
|
48
49
|
|
|
49
50
|
@property
|
|
50
51
|
def nb_messages(self):
|
|
51
|
-
|
|
52
|
+
with self.__messages_lock:
|
|
53
|
+
return len(self.__messages)
|
|
52
54
|
|
|
53
55
|
def _extract_messages_from_data(self, data):
|
|
54
56
|
while True:
|
|
@@ -68,6 +70,10 @@ class MessageSocketClient(object):
|
|
|
68
70
|
with self.__messages_lock:
|
|
69
71
|
self.__messages.append(msg)
|
|
70
72
|
|
|
73
|
+
def reset_received_messages(self):
|
|
74
|
+
with self.__messages_lock:
|
|
75
|
+
self.__messages.clear()
|
|
76
|
+
|
|
71
77
|
def read_message(self):
|
|
72
78
|
res = None
|
|
73
79
|
with self.__messages_lock:
|
|
@@ -115,12 +121,14 @@ class MessageTCPNonBlockingSocketClient(TCPNonBlockingSocketClient, MessageSocke
|
|
|
115
121
|
|
|
116
122
|
def _service_connection(self, key, mask, *, read_bufsize=1024, read_kwargs=None, write_kwargs=None):
|
|
117
123
|
# Read from socket & write in socket
|
|
118
|
-
super()._service_connection(key, mask, read_bufsize=read_bufsize, read_kwargs=read_kwargs, write_kwargs=write_kwargs)
|
|
124
|
+
res = super()._service_connection(key, mask, read_bufsize=read_bufsize, read_kwargs=read_kwargs, write_kwargs=write_kwargs)
|
|
119
125
|
|
|
120
126
|
# Extract messages from received data
|
|
121
127
|
if mask & selectors.EVENT_READ:
|
|
122
128
|
with self._data_lock:
|
|
123
129
|
self._extract_messages_from_data(self._data)
|
|
130
|
+
|
|
131
|
+
return res
|
|
124
132
|
|
|
125
133
|
|
|
126
134
|
|
|
@@ -17,10 +17,9 @@ from holado_python.standard_library.socket.socket import SocketClient
|
|
|
17
17
|
import abc
|
|
18
18
|
from holado_multitask.multithreading.loopfunctionthreaded import LoopFunctionThreaded
|
|
19
19
|
import selectors
|
|
20
|
-
import ssl
|
|
21
|
-
import select
|
|
22
20
|
from holado.common.handlers.undefined import undefined_argument
|
|
23
21
|
import time
|
|
22
|
+
import ssl
|
|
24
23
|
|
|
25
24
|
logger = logging.getLogger(__name__)
|
|
26
25
|
|
|
@@ -84,8 +83,10 @@ class NonBlockingSocketClient(SocketClient):
|
|
|
84
83
|
flags = 0
|
|
85
84
|
else:
|
|
86
85
|
flags = read_kwargs.get('flags', 0)
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
try:
|
|
87
|
+
recv_data = sock.recv(read_bufsize, flags)
|
|
88
|
+
except ssl.SSLWantReadError:
|
|
89
|
+
recv_data = None
|
|
89
90
|
if recv_data:
|
|
90
91
|
has_activity = True
|
|
91
92
|
with self._data_lock: # data is self._data
|
|
@@ -97,7 +98,10 @@ class NonBlockingSocketClient(SocketClient):
|
|
|
97
98
|
with self._data_lock: # data is self._data
|
|
98
99
|
if data.out_bytes:
|
|
99
100
|
has_activity = True
|
|
100
|
-
|
|
101
|
+
try:
|
|
102
|
+
sent = sock.send(data.out_bytes)
|
|
103
|
+
except ssl.SSLWantWriteError:
|
|
104
|
+
sent = 0
|
|
101
105
|
if sent > 0:
|
|
102
106
|
data.out_bytes = data.out_bytes[sent:]
|
|
103
107
|
if logger.isEnabledFor(logging.DEBUG):
|
|
@@ -105,17 +109,6 @@ class NonBlockingSocketClient(SocketClient):
|
|
|
105
109
|
|
|
106
110
|
return has_activity
|
|
107
111
|
|
|
108
|
-
def do_ssl_handshake(self):
|
|
109
|
-
if self.is_with_ssl:
|
|
110
|
-
while True:
|
|
111
|
-
try:
|
|
112
|
-
self.internal_socket.do_handshake()
|
|
113
|
-
break
|
|
114
|
-
except ssl.SSLWantReadError:
|
|
115
|
-
select.select([self.internal_socket], [], [])
|
|
116
|
-
except ssl.SSLWantWriteError:
|
|
117
|
-
select.select([], [self.internal_socket], [])
|
|
118
|
-
|
|
119
112
|
|
|
120
113
|
class TCPNonBlockingSocketClient(NonBlockingSocketClient):
|
|
121
114
|
"""
|
|
@@ -126,16 +119,23 @@ class TCPNonBlockingSocketClient(NonBlockingSocketClient):
|
|
|
126
119
|
super().__init__(name=name, create_ipv4_socket_kwargs=create_ipv4_socket_kwargs, idle_sleep_delay=idle_sleep_delay, do_run_with_recv=do_run_with_recv, do_run_with_send=do_run_with_send)
|
|
127
120
|
|
|
128
121
|
def create_ipv4_socket(self, host, port, **kwargs):
|
|
129
|
-
|
|
122
|
+
socket_kwargs = self._new_ssl_context_if_required(**kwargs)
|
|
130
123
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
124
|
+
if self.is_with_ssl:
|
|
125
|
+
# sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
126
|
+
# sock = Socket.create_connection((host, port), do_connect=False, **socket_kwargs)
|
|
127
|
+
sock = socket.create_connection((host, port), **socket_kwargs)
|
|
128
|
+
sock.setblocking(False)
|
|
129
|
+
|
|
130
|
+
wrap_socket_kwargs = self._ssl_wrap_socket_kwargs
|
|
131
|
+
do_handshake_on_connect = wrap_socket_kwargs.pop('do_handshake_on_connect', False)
|
|
132
|
+
sock = self._ssl_context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=do_handshake_on_connect, **wrap_socket_kwargs)
|
|
133
|
+
self._set_internal_socket(sock, is_ssl_handshake_done_on_connect=do_handshake_on_connect)
|
|
134
|
+
|
|
135
|
+
# sock.connect((host, port))
|
|
138
136
|
else:
|
|
137
|
+
sock = socket.create_connection((host, port), **socket_kwargs)
|
|
138
|
+
sock.setblocking(False)
|
|
139
139
|
self._set_internal_socket(sock)
|
|
140
140
|
|
|
141
141
|
# Register socket
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import logging
|
|
15
15
|
import socket
|
|
16
|
-
from _socket import SHUT_RDWR
|
|
16
|
+
from _socket import SHUT_RDWR, SOCK_STREAM, error
|
|
17
17
|
from holado.common.handlers.object import DeleteableObject
|
|
18
18
|
import abc
|
|
19
19
|
from holado_core.common.exceptions.functional_exception import FunctionalException
|
|
@@ -23,6 +23,10 @@ import threading
|
|
|
23
23
|
import types
|
|
24
24
|
from abc import abstractmethod
|
|
25
25
|
from holado.common.handlers.undefined import undefined_argument
|
|
26
|
+
import copy
|
|
27
|
+
from holado_python.standard_library.typing import Typing
|
|
28
|
+
import ssl
|
|
29
|
+
import select
|
|
26
30
|
|
|
27
31
|
logger = logging.getLogger(__name__)
|
|
28
32
|
|
|
@@ -35,6 +39,62 @@ class Socket(DeleteableObject):
|
|
|
35
39
|
"""
|
|
36
40
|
__metaclass__ = abc.ABCMeta
|
|
37
41
|
|
|
42
|
+
@classmethod
|
|
43
|
+
def create(cls, name, internal_socket):
|
|
44
|
+
res = cls.__new__(cls, name=name)
|
|
45
|
+
res.__init__(name=name)
|
|
46
|
+
res._set_internal_socket(internal_socket)
|
|
47
|
+
return res
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def create_connection(cls, address, timeout=undefined_argument,
|
|
51
|
+
source_address=None, do_connect=True):
|
|
52
|
+
"""Connect to *address* and return the socket object.
|
|
53
|
+
|
|
54
|
+
Convenience function. Connect to *address* (a 2-tuple ``(host,
|
|
55
|
+
port)``) and return the socket object. Passing the optional
|
|
56
|
+
*timeout* parameter will set the timeout on the socket instance
|
|
57
|
+
before attempting to connect. If no *timeout* is supplied, the
|
|
58
|
+
global default timeout setting returned by :func:`getdefaulttimeout`
|
|
59
|
+
is used. If *source_address* is set it must be a tuple of (host, port)
|
|
60
|
+
for the socket to bind as a source address before making the connection.
|
|
61
|
+
A host of '' or port 0 tells the OS to use the default.
|
|
62
|
+
|
|
63
|
+
Note: Implementation is a copy of socket.create_connection, but allowing to not connect.
|
|
64
|
+
If do_connect==True, implementation is identical to socket.create_connection.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
host, port = address
|
|
68
|
+
err = None
|
|
69
|
+
for res in socket.getaddrinfo(host, port, 0, SOCK_STREAM):
|
|
70
|
+
af, socktype, proto, canonname, sa = res # @UnusedVariable
|
|
71
|
+
sock = None
|
|
72
|
+
try:
|
|
73
|
+
sock = socket.socket(af, socktype, proto)
|
|
74
|
+
if timeout is not undefined_argument:
|
|
75
|
+
sock.settimeout(timeout)
|
|
76
|
+
if source_address:
|
|
77
|
+
sock.bind(source_address)
|
|
78
|
+
if do_connect:
|
|
79
|
+
sock.connect(sa)
|
|
80
|
+
# Break explicitly a reference cycle
|
|
81
|
+
err = None
|
|
82
|
+
return sock
|
|
83
|
+
|
|
84
|
+
except error as _:
|
|
85
|
+
err = _
|
|
86
|
+
if sock is not None:
|
|
87
|
+
sock.close()
|
|
88
|
+
|
|
89
|
+
if err is not None:
|
|
90
|
+
try:
|
|
91
|
+
raise err
|
|
92
|
+
finally:
|
|
93
|
+
# Break explicitly a reference cycle
|
|
94
|
+
err = None
|
|
95
|
+
else:
|
|
96
|
+
raise error("getaddrinfo returns an empty list")
|
|
97
|
+
|
|
38
98
|
def __init__(self, *, name=None, create_ipv4_socket_kwargs=None, idle_sleep_delay=undefined_argument):
|
|
39
99
|
"""Socket constructor
|
|
40
100
|
@param name: Socket name
|
|
@@ -44,7 +104,7 @@ class Socket(DeleteableObject):
|
|
|
44
104
|
if name is None and create_ipv4_socket_kwargs is not None:
|
|
45
105
|
if not set(create_ipv4_socket_kwargs.keys()).issuperset({'host', 'port'}):
|
|
46
106
|
raise FunctionalException(f"Parameters 'host' and 'port' must be defined")
|
|
47
|
-
name = f"{create_ipv4_socket_kwargs['host']}:{create_ipv4_socket_kwargs['port']}"
|
|
107
|
+
name = f"{Typing.get_object_class_name(self)}({create_ipv4_socket_kwargs['host']}:{create_ipv4_socket_kwargs['port']})"
|
|
48
108
|
|
|
49
109
|
super().__init__(name)
|
|
50
110
|
self.__socket = None
|
|
@@ -53,7 +113,8 @@ class Socket(DeleteableObject):
|
|
|
53
113
|
self.__idle_sleep_delay = idle_sleep_delay if idle_sleep_delay is not undefined_argument else 0.01
|
|
54
114
|
|
|
55
115
|
# SSL management
|
|
56
|
-
self.
|
|
116
|
+
self.__ssl_kwargs_tuple = None
|
|
117
|
+
self.__ssl_context = None
|
|
57
118
|
self.__is_ssl_handshake_done = False
|
|
58
119
|
|
|
59
120
|
if create_ipv4_socket_kwargs is not None:
|
|
@@ -78,15 +139,30 @@ class Socket(DeleteableObject):
|
|
|
78
139
|
|
|
79
140
|
@property
|
|
80
141
|
def is_with_ssl(self):
|
|
81
|
-
return self.
|
|
142
|
+
return self.__ssl_context is not None
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def _ssl_kwargs(self):
|
|
146
|
+
return copy.copy(self.__ssl_kwargs_tuple[0]) if self.__ssl_kwargs_tuple and self.__ssl_kwargs_tuple[0] is not None else None
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def _ssl_context_kwargs(self):
|
|
150
|
+
return copy.copy(self.__ssl_kwargs_tuple[1]) if self.__ssl_kwargs_tuple and self.__ssl_kwargs_tuple[1] is not None else None
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def _ssl_wrap_socket_kwargs(self):
|
|
154
|
+
return copy.copy(self.__ssl_kwargs_tuple[2]) if self.__ssl_kwargs_tuple and self.__ssl_kwargs_tuple[2] is not None else None
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def _ssl_context(self):
|
|
158
|
+
return self.__ssl_context
|
|
82
159
|
|
|
83
160
|
@property
|
|
84
161
|
def _idle_sleep_delay(self):
|
|
85
162
|
return self.__idle_sleep_delay
|
|
86
163
|
|
|
87
|
-
def _set_internal_socket(self, socket,
|
|
164
|
+
def _set_internal_socket(self, socket, is_ssl_handshake_done_on_connect=True):
|
|
88
165
|
self.__socket = socket
|
|
89
|
-
self.__is_with_ssl = is_with_ssl
|
|
90
166
|
self.__is_ssl_handshake_done = is_ssl_handshake_done_on_connect
|
|
91
167
|
|
|
92
168
|
@abc.abstractmethod
|
|
@@ -98,7 +174,9 @@ class Socket(DeleteableObject):
|
|
|
98
174
|
try:
|
|
99
175
|
self.__socket.shutdown(SHUT_RDWR)
|
|
100
176
|
except OSError as exc:
|
|
101
|
-
if 'Errno
|
|
177
|
+
if 'Errno 9' in str(exc):
|
|
178
|
+
logger.info(f"Got following error on socket shutdown (known to appear when connection is not open correctly or already closed): {exc}")
|
|
179
|
+
elif 'Errno 107' in str(exc):
|
|
102
180
|
logger.info(f"Got following error on socket shutdown (known to appear sometimes during shutdown): {exc}")
|
|
103
181
|
else:
|
|
104
182
|
raise exc
|
|
@@ -149,19 +227,45 @@ class Socket(DeleteableObject):
|
|
|
149
227
|
logger.debug(f"[{self.name}] Sent {res}/{len(data_bytes)} bytes of [{data_bytes}] (type: {type(data_bytes)})")
|
|
150
228
|
return res
|
|
151
229
|
|
|
152
|
-
def
|
|
153
|
-
"""
|
|
230
|
+
def _extract_ssl_kwargs(self, socket_kwargs):
|
|
231
|
+
"""Pop from socket kwargs: global ssl kwargs, SSLContext kwargs, and wrap_socket kwargs.
|
|
154
232
|
"""
|
|
155
|
-
|
|
233
|
+
ssl_kwargs, context_kwargs, wrap_socket_kwargs = None, {}, {}
|
|
156
234
|
if Tools.has_sub_kwargs(socket_kwargs, 'ssl.'):
|
|
157
235
|
ssl_kwargs = Tools.pop_sub_kwargs(socket_kwargs, 'ssl.')
|
|
158
|
-
|
|
159
|
-
|
|
236
|
+
if Tools.has_sub_kwargs(ssl_kwargs, 'context.'):
|
|
237
|
+
context_kwargs = Tools.pop_sub_kwargs(ssl_kwargs, 'context.')
|
|
238
|
+
if Tools.has_sub_kwargs(ssl_kwargs, 'wrap_socket.'):
|
|
239
|
+
wrap_socket_kwargs = Tools.pop_sub_kwargs(ssl_kwargs, 'wrap_socket.')
|
|
240
|
+
self.__ssl_kwargs_tuple = (ssl_kwargs, context_kwargs, wrap_socket_kwargs)
|
|
241
|
+
return socket_kwargs
|
|
242
|
+
|
|
243
|
+
def _new_ssl_context_if_required(self, server_side=False, **kwargs):
|
|
244
|
+
"""Return the remaining socket kwargs after ssl kwargs extraction.
|
|
245
|
+
"""
|
|
246
|
+
res = self._extract_ssl_kwargs(kwargs)
|
|
247
|
+
|
|
248
|
+
if self._ssl_kwargs is not None:
|
|
249
|
+
self.__ssl_context = SslManager.new_ssl_context(server_side=server_side, ssl_kwargs=self._ssl_kwargs, context_kwargs=self._ssl_context_kwargs)
|
|
250
|
+
|
|
251
|
+
return res
|
|
160
252
|
|
|
161
253
|
def do_ssl_handshake(self):
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
254
|
+
if self.is_with_ssl:
|
|
255
|
+
if self.internal_socket.getblocking():
|
|
256
|
+
self.internal_socket.do_handshake()
|
|
257
|
+
# self.internal_socket.do_handshake(block=True)
|
|
258
|
+
else:
|
|
259
|
+
while True:
|
|
260
|
+
try:
|
|
261
|
+
self.internal_socket.do_handshake()
|
|
262
|
+
break
|
|
263
|
+
except ssl.SSLWantReadError:
|
|
264
|
+
select.select([self.internal_socket], [], [])
|
|
265
|
+
except ssl.SSLWantWriteError:
|
|
266
|
+
select.select([], [self.internal_socket], [])
|
|
267
|
+
|
|
268
|
+
|
|
165
269
|
|
|
166
270
|
def ensure_ssl_handshake_is_done(self):
|
|
167
271
|
if self.is_with_ssl and not self.__is_ssl_handshake_done:
|
|
@@ -186,8 +290,8 @@ class SocketClient(Socket):
|
|
|
186
290
|
# Data used in start processing
|
|
187
291
|
self.__data_lock = threading.Lock()
|
|
188
292
|
self.__data = types.SimpleNamespace(
|
|
189
|
-
in_bytes=b
|
|
190
|
-
out_bytes=b
|
|
293
|
+
in_bytes = b'',
|
|
294
|
+
out_bytes = b'',
|
|
191
295
|
)
|
|
192
296
|
self.__with_recv = do_run_with_recv
|
|
193
297
|
self.__with_send = do_run_with_send
|
|
@@ -222,12 +326,22 @@ class SocketClient(Socket):
|
|
|
222
326
|
# Start thread
|
|
223
327
|
super()._start_thread(thread)
|
|
224
328
|
|
|
329
|
+
@property
|
|
330
|
+
def received_data(self):
|
|
331
|
+
with self.__data_lock:
|
|
332
|
+
res = copy.copy(self.__data.in_bytes)
|
|
333
|
+
return res
|
|
334
|
+
|
|
225
335
|
@property
|
|
226
336
|
def received_data_size(self):
|
|
227
337
|
with self.__data_lock:
|
|
228
338
|
res = len(self.__data.in_bytes)
|
|
229
339
|
return res
|
|
230
340
|
|
|
341
|
+
def reset_received_data(self):
|
|
342
|
+
with self.__data_lock:
|
|
343
|
+
self.__data.in_bytes = b''
|
|
344
|
+
|
|
231
345
|
def read(self, bufsize=1024, **kwargs):
|
|
232
346
|
if self.is_started and self.is_run_with_recv:
|
|
233
347
|
with self.__data_lock:
|
|
@@ -263,8 +377,7 @@ class SocketServer(Socket):
|
|
|
263
377
|
|
|
264
378
|
def accept(self):
|
|
265
379
|
conn, addr = self.internal_socket.accept()
|
|
266
|
-
res = Socket(name=f"[{self.name}] Connection with {addr}")
|
|
267
|
-
res._set_internal_socket(conn)
|
|
380
|
+
res = Socket.create(name=f"[{self.name}] Connection with {addr}", internal_socket=conn)
|
|
268
381
|
return res, addr
|
|
269
382
|
|
|
270
383
|
@abc.abstractmethod
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
localhost.
|
|
1
|
+
localhost.crt and localhost.key were generated with command:
|
|
2
2
|
openssl req -x509 -out localhost.crt -keyout localhost.key \
|
|
3
3
|
-newkey rsa:2048 -nodes -sha256 \
|
|
4
4
|
-subj '/CN=localhost' -extensions EXT -config <( \
|