holado 0.4.2__py3-none-any.whl → 0.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of holado might be problematic. Click here for more details.
- holado/common/context/session_context.py +6 -0
- {holado-0.4.2.dist-info → holado-0.5.2.dist-info}/METADATA +1 -1
- {holado-0.4.2.dist-info → holado-0.5.2.dist-info}/RECORD +57 -28
- holado_ais/ais/ais_messages.py +44 -15
- holado_ais/ais/patch_pyais.py +47 -176
- holado_ais/tests/behave/steps/ais/ais_messages_steps.py +2 -6
- holado_core/common/resource/persisted_data_manager.py +4 -5
- holado_core/common/resource/resource_manager.py +4 -18
- holado_core/common/tools/converters/converter.py +14 -3
- holado_core/common/tools/tools.py +9 -2
- holado_data/data/generator/generator_manager.py +1 -13
- holado_db/tools/db/clients/base/db_audit.py +94 -0
- holado_db/tools/db/clients/base/db_client.py +145 -59
- holado_db/tools/db/clients/postgresql/postgresql_audit.py +75 -0
- holado_db/tools/db/clients/postgresql/postgresql_client.py +4 -0
- holado_db/tools/db/clients/sqlite/sqlite_audit.py +70 -0
- holado_db/tools/db/clients/sqlite/sqlite_client.py +4 -0
- holado_django/server/patch_djangogrpcframework.py +46 -0
- holado_logging/common/logging/holado_logger.py +1 -6
- holado_multitask/multithreading/thread.py +13 -7
- holado_multitask/multithreading/timer.py +3 -0
- holado_python/common/tools/datetime.py +31 -13
- holado_python/standard_library/ssl/resources/certificates/tcpbin.crt +16 -16
- holado_python/standard_library/ssl/resources/certificates/tcpbin.key +26 -26
- holado_rabbitmq/tools/rabbitmq/rabbitmq_blocking_client.py +24 -2
- holado_rabbitmq/tools/rabbitmq/rabbitmq_client.py +2 -2
- holado_report/report/builders/failure_report_builder.py +129 -0
- holado_report/report/report_manager.py +4 -0
- holado_rest/api/rest/rest_client.py +19 -7
- holado_rest/tests/behave/steps/api/rest_client_steps.py +1 -2
- test_holado/features/NonReg/holado_ais/message_types/type-10.feature +38 -0
- test_holado/features/NonReg/holado_ais/message_types/type-12.feature +37 -0
- test_holado/features/NonReg/holado_ais/message_types/type-14.feature +36 -0
- test_holado/features/NonReg/holado_ais/message_types/type-15.feature +36 -0
- test_holado/features/NonReg/holado_ais/message_types/type-16.feature +38 -0
- test_holado/features/NonReg/holado_ais/message_types/type-17.feature +46 -0
- test_holado/features/NonReg/holado_ais/message_types/type-18.feature +37 -0
- test_holado/features/NonReg/holado_ais/message_types/type-19.feature +38 -0
- test_holado/features/NonReg/holado_ais/message_types/type-1_2_3.feature +42 -0
- test_holado/features/NonReg/holado_ais/message_types/type-20.feature +38 -0
- test_holado/features/NonReg/holado_ais/message_types/type-21.feature +37 -0
- test_holado/features/NonReg/holado_ais/message_types/type-22.feature +84 -0
- test_holado/features/NonReg/holado_ais/message_types/type-23.feature +49 -0
- test_holado/features/NonReg/holado_ais/message_types/type-24.feature +72 -0
- test_holado/features/NonReg/holado_ais/message_types/type-25.feature +143 -0
- test_holado/features/NonReg/holado_ais/message_types/type-26.feature +144 -0
- test_holado/features/NonReg/holado_ais/message_types/type-27.feature +36 -0
- test_holado/features/NonReg/holado_ais/message_types/type-4_11.feature +39 -0
- test_holado/features/NonReg/holado_ais/message_types/type-5.feature +33 -0
- test_holado/features/NonReg/holado_ais/message_types/type-6.feature +37 -0
- test_holado/features/NonReg/holado_ais/message_types/type-7_13.feature +43 -0
- test_holado/features/NonReg/holado_ais/message_types/type-8.feature +37 -0
- test_holado/features/NonReg/holado_ais/message_types/type-9.feature +37 -0
- test_holado/tools/django/api_grpc/manage.py +2 -0
- test_holado/tools/django/api_grpc/patch_djangogrpcframework.py +42 -0
- {holado-0.4.2.dist-info → holado-0.5.2.dist-info}/WHEEL +0 -0
- {holado-0.4.2.dist-info → holado-0.5.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -41,6 +41,8 @@ class DBClient(object):
|
|
|
41
41
|
|
|
42
42
|
self.__connection = None
|
|
43
43
|
self.__query_builder = None
|
|
44
|
+
self.__audit_manager = None
|
|
45
|
+
self.__column_names_by_table = {}
|
|
44
46
|
|
|
45
47
|
@property
|
|
46
48
|
def name(self):
|
|
@@ -68,6 +70,19 @@ class DBClient(object):
|
|
|
68
70
|
raise TechnicalException(f"DB client '{self.__name}' doesn't manage cursor")
|
|
69
71
|
return self.__cursor
|
|
70
72
|
|
|
73
|
+
@property
|
|
74
|
+
def audit_manager(self):
|
|
75
|
+
if self.__audit_manager is None:
|
|
76
|
+
self.__audit_manager = self._new_audit_manager()
|
|
77
|
+
return self.__audit_manager
|
|
78
|
+
|
|
79
|
+
def _new_audit_manager(self):
|
|
80
|
+
"""Create a new audit manager.
|
|
81
|
+
By default, use a audit manager with a single table of audit for whole DB and with triggers on audited tables.
|
|
82
|
+
Override this method in order to define which audit manager to use.
|
|
83
|
+
"""
|
|
84
|
+
raise NotImplementedError()
|
|
85
|
+
|
|
71
86
|
def connect(self):
|
|
72
87
|
try:
|
|
73
88
|
self.__connection = self._connect(**self.__connect_kwargs)
|
|
@@ -83,22 +98,45 @@ class DBClient(object):
|
|
|
83
98
|
if not self.is_connected:
|
|
84
99
|
raise FunctionalException(f"DB Client '{self.name}' is not connected")
|
|
85
100
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
# Manage queries
|
|
102
|
+
|
|
103
|
+
def insert(self, table_name, data: dict, do_commit=None):
|
|
104
|
+
"""
|
|
105
|
+
Insert given data.
|
|
106
|
+
Parameter 'data' has to be a dictionary with keys equal to table column names.
|
|
107
|
+
"""
|
|
108
|
+
query, values = self.query_builder.insert(table_name, data)
|
|
109
|
+
return self.execute_query(query, *values, do_commit=do_commit)
|
|
110
|
+
|
|
111
|
+
def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None, do_commit=None):
|
|
112
|
+
"""
|
|
113
|
+
Update given data.
|
|
114
|
+
Parameters 'data' and 'where_data' have to be dictionaries with keys equal to table column names.
|
|
115
|
+
"""
|
|
116
|
+
query, values = self.query_builder.update(table_name, data, where_data=where_data, where_compare_data=where_compare_data)
|
|
117
|
+
return self.execute_query(query, *values, do_commit=do_commit)
|
|
118
|
+
|
|
119
|
+
def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*", **kwargs):
|
|
120
|
+
"""
|
|
121
|
+
Select by filtering on given where data.
|
|
122
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
123
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
124
|
+
"""
|
|
125
|
+
query, values = self.query_builder.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=sql_return)
|
|
126
|
+
return self.execute_query(query, *values, do_commit=False, **kwargs)
|
|
127
|
+
|
|
128
|
+
def delete(self, table_name, where_data: dict=None, where_compare_data: list=None, do_commit=None):
|
|
129
|
+
"""
|
|
130
|
+
Delete by filtering on given where data.
|
|
131
|
+
Parameter 'where_data' has to be a dictionary with keys equal to table column names.
|
|
132
|
+
"""
|
|
133
|
+
query, values = self.query_builder.delete(table_name, where_data=where_data, where_compare_data=where_compare_data)
|
|
134
|
+
return self.execute_query(query, *values, do_commit=do_commit)
|
|
101
135
|
|
|
136
|
+
def count(self, table_name, where_data: dict=None, where_compare_data: list=None):
|
|
137
|
+
result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return="count(*)")
|
|
138
|
+
return result[0][0].content
|
|
139
|
+
|
|
102
140
|
def execute_query(self, query, *args, **kwargs):
|
|
103
141
|
sql = self.query_builder.to_sql(query)
|
|
104
142
|
|
|
@@ -126,7 +164,7 @@ class DBClient(object):
|
|
|
126
164
|
|
|
127
165
|
# Execute
|
|
128
166
|
if Tools.do_log(logger, logging.DEBUG):
|
|
129
|
-
logger.debug(f"[{self.name}] Executing SQL [{sql}]
|
|
167
|
+
logger.debug(f"[{self.name}] Executing SQL [{Tools.truncate_text(sql, 1000)}] (args: {Tools.truncate_text(args, 1000)} ; kwargs: {Tools.truncate_text(kwargs, 1000)})...")
|
|
130
168
|
try:
|
|
131
169
|
if args:
|
|
132
170
|
self.cursor.execute(sql, args)
|
|
@@ -172,7 +210,7 @@ class DBClient(object):
|
|
|
172
210
|
else:
|
|
173
211
|
res = None
|
|
174
212
|
|
|
175
|
-
self.__log_sql_result(res, message=f"Executed SQL [{sql}]
|
|
213
|
+
self.__log_sql_result(res, message=f"Executed SQL [{Tools.truncate_text(sql, 1000)}] (args: {Tools.truncate_text(args, 1000)} ; kwargs: {Tools.truncate_text(kwargs, 1000)})")
|
|
176
214
|
|
|
177
215
|
# Manage commit
|
|
178
216
|
if do_commit:
|
|
@@ -184,6 +222,52 @@ class DBClient(object):
|
|
|
184
222
|
|
|
185
223
|
return res
|
|
186
224
|
|
|
225
|
+
def is_query_type(self, sql_or_query, query_type):
|
|
226
|
+
if isinstance(sql_or_query, pypika.queries.QueryBuilder):
|
|
227
|
+
sql = self.query_builder.to_sql(sql_or_query)
|
|
228
|
+
else:
|
|
229
|
+
sql = sql_or_query
|
|
230
|
+
if not isinstance(sql, str):
|
|
231
|
+
raise TechnicalException(f"Unexpected type '{Typing.get_object_class_fullname(sql)}' for parameter sql_or_query")
|
|
232
|
+
sql = sql.lower()
|
|
233
|
+
|
|
234
|
+
# Parsing query with sql_metadata can be costly, thus begin by fast and simple methods
|
|
235
|
+
query_type_str = query_type.lower() if isinstance(query_type, str) else query_type.name.lower()
|
|
236
|
+
for qt in ['select', 'insert', 'update', 'delete']:
|
|
237
|
+
if sql.startswith(qt):
|
|
238
|
+
return qt == query_type_str
|
|
239
|
+
|
|
240
|
+
# Parse more complex queries
|
|
241
|
+
if isinstance(query_type, str):
|
|
242
|
+
query_type = QueryType[query_type.upper()]
|
|
243
|
+
if not isinstance(query_type, QueryType):
|
|
244
|
+
raise TechnicalException(f"Unmanage query_type of type '{Typing.get_object_class_fullname(query_type)}'")
|
|
245
|
+
|
|
246
|
+
try:
|
|
247
|
+
p = sql_metadata.Parser(sql)
|
|
248
|
+
res = p.query_type == query_type
|
|
249
|
+
except ValueError as exc:
|
|
250
|
+
if "Not supported query type" in str(exc):
|
|
251
|
+
res = False
|
|
252
|
+
else:
|
|
253
|
+
raise TechnicalException(f"Failed to define query type") from exc
|
|
254
|
+
except Exception as exc:
|
|
255
|
+
raise TechnicalException(f"Failed to define query type") from exc
|
|
256
|
+
|
|
257
|
+
return res
|
|
258
|
+
|
|
259
|
+
def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None, where_compare_data: list=None):
|
|
260
|
+
"""
|
|
261
|
+
Set or update a json field with key=value.
|
|
262
|
+
"""
|
|
263
|
+
raise NotImplementedError()
|
|
264
|
+
|
|
265
|
+
def _get_sql_placeholder(self):
|
|
266
|
+
"""
|
|
267
|
+
Return the character/string to use as placeholder in SQL requests.
|
|
268
|
+
"""
|
|
269
|
+
raise NotImplementedError()
|
|
270
|
+
|
|
187
271
|
def _get_base_exception_type(self):
|
|
188
272
|
raise NotImplementedError()
|
|
189
273
|
|
|
@@ -203,57 +287,59 @@ class DBClient(object):
|
|
|
203
287
|
else:
|
|
204
288
|
return str(sql_result)
|
|
205
289
|
|
|
290
|
+
|
|
291
|
+
# Manage transactions
|
|
292
|
+
|
|
206
293
|
def commit(self):
|
|
207
294
|
self.connection.commit()
|
|
208
295
|
|
|
209
296
|
def rollback(self):
|
|
210
297
|
self.connection.rollback()
|
|
211
|
-
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
# Manage tables
|
|
301
|
+
|
|
212
302
|
def exist_table(self, table_name):
|
|
213
303
|
raise NotImplementedError()
|
|
214
304
|
|
|
215
|
-
def
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
Parameters 'data' and 'where_data' have to be dictionaries with keys equal to table column names.
|
|
227
|
-
"""
|
|
228
|
-
query, values = self.query_builder.update(table_name, data, where_data=where_data, where_compare_data=where_compare_data)
|
|
229
|
-
return self.execute_query(query, *values, do_commit=do_commit)
|
|
305
|
+
def create_table(self, table_name, create_sql, raise_if_exist=False, do_commit=True, do_audit=False):
|
|
306
|
+
if not self.exist_table(table_name):
|
|
307
|
+
self.execute(create_sql, do_commit=do_commit)
|
|
308
|
+
if not self.exist_table(table_name):
|
|
309
|
+
raise TechnicalException(f"Failed to create table '{table_name}' with SQL request [{create_sql}]")
|
|
310
|
+
|
|
311
|
+
# Audit table
|
|
312
|
+
if do_audit:
|
|
313
|
+
self.audit_manager.audit_table(table_name)
|
|
314
|
+
elif raise_if_exist:
|
|
315
|
+
raise FunctionalException(f"Table '{table_name}' already exists")
|
|
230
316
|
|
|
231
|
-
def
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
317
|
+
def drop_table(self, table_name, raise_if_not_exist=False, do_commit=True):
|
|
318
|
+
if self.exist_table(table_name):
|
|
319
|
+
sql = f"drop table {table_name};"
|
|
320
|
+
self.execute(sql, do_commit=do_commit)
|
|
321
|
+
if self.exist_table(table_name):
|
|
322
|
+
raise TechnicalException(f"Failed to drop table '{table_name}' with SQL request [{sql}]")
|
|
323
|
+
elif raise_if_not_exist:
|
|
324
|
+
raise FunctionalException(f"Table '{table_name}' doesn't exist")
|
|
239
325
|
|
|
240
|
-
def
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
"""
|
|
245
|
-
query, values = self.query_builder.delete(table_name, where_data=where_data, where_compare_data=where_compare_data)
|
|
246
|
-
return self.execute_query(query, *values, do_commit=do_commit)
|
|
326
|
+
def get_table_column_names(self, table_name):
|
|
327
|
+
if table_name not in self.__column_names_by_table:
|
|
328
|
+
self.__column_names_by_table[table_name] = self._get_table_column_names(table_name)
|
|
329
|
+
return self.__column_names_by_table[table_name]
|
|
247
330
|
|
|
248
|
-
def
|
|
249
|
-
""
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
331
|
+
def _get_table_column_names(self, table_name):
|
|
332
|
+
sql = f"select * from {table_name} limit 1"
|
|
333
|
+
try:
|
|
334
|
+
self.cursor.execute(sql)
|
|
335
|
+
except self._get_base_exception_type() as exc:
|
|
336
|
+
self.rollback()
|
|
337
|
+
raise TechnicalException(f"[{self.name}] Error while executing SQL [{sql}]") from exc
|
|
338
|
+
|
|
339
|
+
if self.cursor.description:
|
|
340
|
+
res = [field[0] for field in self.cursor.description]
|
|
341
|
+
else:
|
|
342
|
+
raise TechnicalException(f"Failed to get column names of table {table_name}")
|
|
343
|
+
|
|
344
|
+
return res
|
|
253
345
|
|
|
254
|
-
def _get_sql_placeholder(self):
|
|
255
|
-
"""
|
|
256
|
-
Return the character/string to use as placeholder in SQL requests.
|
|
257
|
-
"""
|
|
258
|
-
raise NotImplementedError()
|
|
259
|
-
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
|
|
2
|
+
#################################################
|
|
3
|
+
# HolAdo (Holistic Automation do)
|
|
4
|
+
#
|
|
5
|
+
# (C) Copyright 2023 by Eric Klumpp
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
+
#
|
|
9
|
+
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
+
|
|
11
|
+
# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
|
|
12
|
+
#################################################
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
from holado.common.handlers.undefined import default_value
|
|
16
|
+
from holado_db.tools.db.clients.base.db_audit import TriggerAuditManager
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PostgreSQLTriggerAuditManager(TriggerAuditManager):
|
|
22
|
+
"""Manage audit of tables by triggers.
|
|
23
|
+
WARNING: This case is not tested yet
|
|
24
|
+
"""
|
|
25
|
+
def __init__(self, name, db_client):
|
|
26
|
+
super().__init__(name, db_client)
|
|
27
|
+
|
|
28
|
+
def _get_audit_table_sql_create(self, audit_table_name=default_value):
|
|
29
|
+
audit_table_name = self._get_audit_table_name(audit_table_name)
|
|
30
|
+
return f"""CREATE TABLE {audit_table_name} (
|
|
31
|
+
id SERIAL PRIMARY KEY,
|
|
32
|
+
table_name TEXT,
|
|
33
|
+
record_id INTEGER,
|
|
34
|
+
operation_type TEXT,
|
|
35
|
+
changed_at TIMESTAMP DEFAULT now(),
|
|
36
|
+
changed_by TEXT,
|
|
37
|
+
previous_values JSONB,
|
|
38
|
+
new_values JSONB
|
|
39
|
+
)"""
|
|
40
|
+
|
|
41
|
+
def _get_drop_trigger_sql_of_audit_table_operation(self, table_name, operation_type, audit_table_name):
|
|
42
|
+
audit_table_name = self._get_audit_table_name(audit_table_name)
|
|
43
|
+
trigger_name = self.get_trigger_name(table_name, operation_type)
|
|
44
|
+
return f"DROP TRIGGER IF EXISTS {trigger_name} on {table_name}"
|
|
45
|
+
|
|
46
|
+
def _get_create_trigger_sql_to_audit_table_operation(self, table_name, operation_type, audit_table_name):
|
|
47
|
+
audit_table_name = self._get_audit_table_name(audit_table_name)
|
|
48
|
+
trigger_name = self.get_trigger_name(table_name, operation_type)
|
|
49
|
+
trigger_insert_sql = self.__get_trigger_insert_sql(table_name, operation_type, audit_table_name)
|
|
50
|
+
return f"""CREATE TRIGGER {trigger_name} AFTER {operation_type.upper()} ON {table_name}
|
|
51
|
+
BEGIN
|
|
52
|
+
{trigger_insert_sql}
|
|
53
|
+
END;
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __get_trigger_insert_sql(self, table_name, operation_type, audit_table_name):
|
|
57
|
+
rowid_txt = "NEW.id" if operation_type in ['insert', 'update'] else "OLD.id"
|
|
58
|
+
op_type = operation_type[0]
|
|
59
|
+
prev_vals_txt = "to_jsonb(OLD)" if operation_type in ['update', 'delete'] else "NULL"
|
|
60
|
+
new_vals_txt = "to_jsonb(NEW)" if operation_type in ['insert', 'update'] else "NULL"
|
|
61
|
+
|
|
62
|
+
audit_info = [f"'{table_name}'",
|
|
63
|
+
rowid_txt,
|
|
64
|
+
f"'{op_type}'",
|
|
65
|
+
"current_user",
|
|
66
|
+
prev_vals_txt,
|
|
67
|
+
new_vals_txt,
|
|
68
|
+
]
|
|
69
|
+
audit_info = ','.join(audit_info)
|
|
70
|
+
|
|
71
|
+
return f"""INSERT INTO {audit_table_name} (table_name, record_id, operation_type, changed_by, previous_values, new_values)
|
|
72
|
+
VALUES ({audit_info});"""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
@@ -17,6 +17,7 @@ from holado_core.common.tables.table_with_header import TableWithHeader
|
|
|
17
17
|
from holado_core.common.tools.tools import Tools
|
|
18
18
|
import json
|
|
19
19
|
from pypika.terms import Function, LiteralValue
|
|
20
|
+
from holado_db.tools.db.clients.postgresql.postgresql_audit import PostgreSQLTriggerAuditManager
|
|
20
21
|
|
|
21
22
|
logger = logging.getLogger(__name__)
|
|
22
23
|
|
|
@@ -43,6 +44,9 @@ class PostgreSQLClient(DBClient):
|
|
|
43
44
|
def __init__(self, name, connect_kwargs):
|
|
44
45
|
super().__init__(name if name else 'PostgreSQL', connect_kwargs)
|
|
45
46
|
|
|
47
|
+
def _new_audit_manager(self):
|
|
48
|
+
return PostgreSQLTriggerAuditManager("Audit with triggers", self)
|
|
49
|
+
|
|
46
50
|
def _get_base_exception_type(self):
|
|
47
51
|
return psycopg.Error
|
|
48
52
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
|
|
2
|
+
#################################################
|
|
3
|
+
# HolAdo (Holistic Automation do)
|
|
4
|
+
#
|
|
5
|
+
# (C) Copyright 2023 by Eric Klumpp
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
+
#
|
|
9
|
+
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
+
|
|
11
|
+
# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
|
|
12
|
+
#################################################
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
from holado.common.handlers.undefined import default_value
|
|
16
|
+
from holado_db.tools.db.clients.base.db_audit import TriggerAuditManager
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SQLite3TriggerAuditManager(TriggerAuditManager):
|
|
22
|
+
def __init__(self, name, db_client):
|
|
23
|
+
super().__init__(name, db_client)
|
|
24
|
+
|
|
25
|
+
def _get_audit_table_sql_create(self, audit_table_name=default_value):
|
|
26
|
+
audit_table_name = self._get_audit_table_name(audit_table_name)
|
|
27
|
+
return f"""CREATE TABLE {audit_table_name} (
|
|
28
|
+
id INTEGER PRIMARY KEY,
|
|
29
|
+
table_name TEXT,
|
|
30
|
+
record_id INTEGER,
|
|
31
|
+
operation_type TEXT,
|
|
32
|
+
changed_at TEXT,
|
|
33
|
+
previous_values TEXT,
|
|
34
|
+
new_values TEXT
|
|
35
|
+
)"""
|
|
36
|
+
|
|
37
|
+
def _get_drop_trigger_sql_of_audit_table_operation(self, table_name, operation_type, audit_table_name):
|
|
38
|
+
trigger_name = self.get_trigger_name(table_name, operation_type)
|
|
39
|
+
return f"DROP TRIGGER IF EXISTS {trigger_name}"
|
|
40
|
+
|
|
41
|
+
def _get_create_trigger_sql_to_audit_table_operation(self, table_name, operation_type, audit_table_name):
|
|
42
|
+
audit_table_name = self._get_audit_table_name(audit_table_name)
|
|
43
|
+
trigger_name = self.get_trigger_name(table_name, operation_type)
|
|
44
|
+
trigger_insert_sql = self.__get_trigger_insert_sql(table_name, operation_type, audit_table_name)
|
|
45
|
+
return f"""CREATE TRIGGER {trigger_name} AFTER {operation_type.upper()} ON {table_name}
|
|
46
|
+
BEGIN
|
|
47
|
+
{trigger_insert_sql}
|
|
48
|
+
END;
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __get_trigger_insert_sql(self, table_name, operation_type, audit_table_name):
|
|
52
|
+
values = [f"'{table_name}'",
|
|
53
|
+
"NEW.rowid" if operation_type in ['insert', 'update'] else "OLD.rowid",
|
|
54
|
+
f"'{operation_type[0]}'",
|
|
55
|
+
"DATETIME('now')",
|
|
56
|
+
self.__get_row_version_json_object_sql(table_name, 'OLD') if operation_type in ['update', 'delete'] else "NULL",
|
|
57
|
+
self.__get_row_version_json_object_sql(table_name, 'NEW') if operation_type in ['insert', 'update'] else "NULL",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
return f"""INSERT INTO {audit_table_name} (table_name, record_id, operation_type, changed_at, previous_values, new_values)
|
|
61
|
+
VALUES ({','.join(values)});"""
|
|
62
|
+
|
|
63
|
+
def __get_row_version_json_object_sql(self, table_name, row_version):
|
|
64
|
+
col_names = self._db_client.get_table_column_names(table_name)
|
|
65
|
+
json_object_args = []
|
|
66
|
+
for cn in col_names:
|
|
67
|
+
json_object_args.append(f"'{cn}'")
|
|
68
|
+
json_object_args.append(f"{row_version}.{cn}")
|
|
69
|
+
return f"json_object({','.join(json_object_args)})"
|
|
70
|
+
|
|
@@ -15,6 +15,7 @@ import logging
|
|
|
15
15
|
from holado_db.tools.db.clients.base.db_client import DBClient
|
|
16
16
|
from holado_core.common.exceptions.functional_exception import FunctionalException
|
|
17
17
|
from holado_core.common.tools.tools import Tools
|
|
18
|
+
from holado_db.tools.db.clients.sqlite.sqlite_audit import SQLite3TriggerAuditManager
|
|
18
19
|
|
|
19
20
|
logger = logging.getLogger(__name__)
|
|
20
21
|
|
|
@@ -35,6 +36,9 @@ class SQLite3Client(DBClient):
|
|
|
35
36
|
def __init__(self, name, connect_kwargs):
|
|
36
37
|
super().__init__(name if name else 'SQLite3', connect_kwargs)
|
|
37
38
|
|
|
39
|
+
def _new_audit_manager(self):
|
|
40
|
+
return SQLite3TriggerAuditManager("Audit with triggers", self)
|
|
41
|
+
|
|
38
42
|
def _get_base_exception_type(self):
|
|
39
43
|
return sqlite3.Error
|
|
40
44
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
|
|
2
|
+
#################################################
|
|
3
|
+
# HolAdo (Holistic Automation do)
|
|
4
|
+
#
|
|
5
|
+
# (C) Copyright 2021-2025 by Eric Klumpp
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
+
#
|
|
9
|
+
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
+
|
|
11
|
+
# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
|
|
12
|
+
#################################################
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
#################################################
|
|
16
|
+
#
|
|
17
|
+
# Patches are done to follow ITU recommendation:
|
|
18
|
+
# https://www.e-navigation.nl/sites/default/files/R-REC-M.1371-5-201402-I!!PDF-E_1.pdf
|
|
19
|
+
#
|
|
20
|
+
#################################################
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# Since Django 4.1, requires_system_checks cannot be a boolean anymore.
|
|
29
|
+
# Framework djangogrpcframework is not maintained anymore.
|
|
30
|
+
# Patch it by removing reference to requires_system_checks
|
|
31
|
+
#
|
|
32
|
+
# USAGE:
|
|
33
|
+
# 1. If not using holado in django project, copy this file in django project
|
|
34
|
+
# 2. In manage.py, import this file
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
import django_grpc_framework # @UnusedImport
|
|
38
|
+
with_django_grpc_framework = True
|
|
39
|
+
except:
|
|
40
|
+
with_django_grpc_framework = True
|
|
41
|
+
|
|
42
|
+
if with_django_grpc_framework:
|
|
43
|
+
from django_grpc_framework.management.commands.grpcrunserver import Command # @UnusedImport
|
|
44
|
+
del Command.requires_system_checks
|
|
45
|
+
|
|
46
|
+
|
|
@@ -43,12 +43,7 @@ class HALogger(logging.Logger):
|
|
|
43
43
|
|
|
44
44
|
if msg_size_limit is None:
|
|
45
45
|
msg_size_limit = HALogger.default_message_size_limit
|
|
46
|
-
|
|
47
|
-
if msg_size_limit is not None and msg_size_limit > 0 and len(msg) > msg_size_limit:
|
|
48
|
-
truncated_suffix = f"[...({len(msg)-msg_size_limit})]"
|
|
49
|
-
msg_to_log = Tools.truncate_text(msg, msg_size_limit, truncated_suffix)
|
|
50
|
-
else:
|
|
51
|
-
msg_to_log = msg
|
|
46
|
+
msg_to_log = Tools.truncate_text(msg, msg_size_limit)
|
|
52
47
|
|
|
53
48
|
logging.Logger._log(self, level, msg_to_log, args, exc_info=exc_info, extra=extra, stack_info=stack_info, stacklevel=stacklevel)
|
|
54
49
|
|
|
@@ -22,20 +22,21 @@ from holado.common.handlers.undefined import undefined_argument, default_context
|
|
|
22
22
|
from holado_core.common.exceptions.technical_exception import TimeoutTechnicalException
|
|
23
23
|
from holado_core.common.exceptions.functional_exception import FunctionalException
|
|
24
24
|
from holado_python.common.tools.datetime import DateTime, FORMAT_DATETIME_ISO
|
|
25
|
+
from holado.common.handlers.object import DeleteableObject
|
|
25
26
|
|
|
26
27
|
logger = logging.getLogger(__name__)
|
|
27
28
|
|
|
28
29
|
|
|
29
|
-
class Thread(threading.Thread):
|
|
30
|
+
class Thread(threading.Thread, DeleteableObject):
|
|
30
31
|
""" Helper class managing a thread."""
|
|
31
32
|
|
|
32
33
|
def __init__(self, name, default_wait_timeout=None, register_thread=True, register_scope=default_context, delay_before_run_sec=None):
|
|
33
|
-
|
|
34
|
+
threading.Thread.__init__(self)
|
|
35
|
+
DeleteableObject.__init__(self, name)
|
|
34
36
|
|
|
35
37
|
if default_wait_timeout is None:
|
|
36
38
|
default_wait_timeout = Config.timeout_seconds * 1000
|
|
37
39
|
|
|
38
|
-
self.__name = name
|
|
39
40
|
# Note: if SessionContext doesn't have an instance (like during session context reset), use name+now as unique name
|
|
40
41
|
self.__unique_name = SessionContext.instance().multitask_manager.prepare_thread(name) if SessionContext.has_instance() else f"{name}_{DateTime.datetime_2_str(DateTime.now(), FORMAT_DATETIME_ISO)}"
|
|
41
42
|
self.__default_wait_timeout = default_wait_timeout
|
|
@@ -51,14 +52,15 @@ class Thread(threading.Thread):
|
|
|
51
52
|
|
|
52
53
|
# Register thread
|
|
53
54
|
# Note: Do not register thread if session context doesn't have an instance (like during session context reset)
|
|
55
|
+
self.__is_registered = False
|
|
56
|
+
self.__register_scope = None
|
|
54
57
|
if register_thread and SessionContext.has_instance():
|
|
55
58
|
# Registered name has to be unique
|
|
56
59
|
SessionContext.instance().threads_manager.register_thread(self.__unique_name, self, scope=register_scope)
|
|
60
|
+
self.__is_registered = True
|
|
61
|
+
self.__register_scope = register_scope
|
|
62
|
+
self.on_delete_call_func(self.__unregister_thread)
|
|
57
63
|
|
|
58
|
-
@property
|
|
59
|
-
def name(self):
|
|
60
|
-
return self.__name
|
|
61
|
-
|
|
62
64
|
@property
|
|
63
65
|
def unique_name(self):
|
|
64
66
|
return self.__unique_name
|
|
@@ -149,6 +151,10 @@ class Thread(threading.Thread):
|
|
|
149
151
|
elif wait_round_counter % 10 == 0:
|
|
150
152
|
logger.debug("[{}] Still waiting {} (round {})".format(self.name, event_details, wait_round_counter))
|
|
151
153
|
logger.debug("[{}] Finished waiting {}...".format(self.name, event_details))
|
|
154
|
+
|
|
155
|
+
def __unregister_thread(self):
|
|
156
|
+
if self.__is_registered:
|
|
157
|
+
SessionContext.instance().threads_manager.unregister_thread(self.__unique_name, self, scope=self.__register_scope)
|
|
152
158
|
|
|
153
159
|
|
|
154
160
|
class InterruptableThread(Thread):
|
|
@@ -40,7 +40,10 @@ class Timer(InterruptableThread):
|
|
|
40
40
|
self.finished.set()
|
|
41
41
|
|
|
42
42
|
def run(self):
|
|
43
|
+
# Wait timer interval or canceled (ie finished==True)
|
|
43
44
|
self.finished.wait(self.interval)
|
|
45
|
+
|
|
46
|
+
# Run function if not canceled
|
|
44
47
|
if not self.finished.is_set():
|
|
45
48
|
self.function(*self.args, **self.kwargs)
|
|
46
49
|
self.finished.set()
|
|
@@ -158,26 +158,51 @@ class DateTime(object):
|
|
|
158
158
|
res = dt_value.strftime(dt_format)
|
|
159
159
|
return res
|
|
160
160
|
|
|
161
|
-
|
|
162
|
-
|
|
161
|
+
@classmethod
|
|
162
|
+
def _get_datetime(cls, dt):
|
|
163
|
+
res = None
|
|
164
|
+
if dt is not None:
|
|
165
|
+
if isinstance(dt, datetime.datetime):
|
|
166
|
+
res = dt
|
|
167
|
+
elif isinstance(dt, str):
|
|
168
|
+
res = DateTime.str_2_datetime(dt)
|
|
169
|
+
else:
|
|
170
|
+
raise TechnicalException(f"Datetime can be a datetime, a str containing a datetime, or None")
|
|
171
|
+
return res
|
|
163
172
|
|
|
164
173
|
@classmethod
|
|
165
174
|
def _get_epoch_datetime(cls, epoch):
|
|
166
175
|
res = None
|
|
167
176
|
if epoch is not None:
|
|
168
|
-
if isinstance(epoch,
|
|
177
|
+
if isinstance(epoch, datetime.datetime):
|
|
178
|
+
res = epoch
|
|
179
|
+
elif isinstance(epoch, str):
|
|
169
180
|
if epoch in ['EPOCH_1970', 'EPOCH_JULIAN_CNES', 'EPOCH_JULIAN_NASA']:
|
|
170
181
|
res = eval(epoch)
|
|
171
182
|
else:
|
|
172
183
|
res = DateTime.str_2_datetime(epoch)
|
|
173
|
-
elif isinstance(epoch, datetime.datetime):
|
|
174
|
-
res = epoch
|
|
175
184
|
else:
|
|
176
185
|
raise TechnicalException(f"Epoch can be a datetime, a str containing a datetime, a str containing an EPOCH name, or None")
|
|
177
186
|
return res
|
|
178
187
|
|
|
188
|
+
@classmethod
|
|
189
|
+
def _get_timestamp(cls, ts):
|
|
190
|
+
res = None
|
|
191
|
+
if ts is not None:
|
|
192
|
+
if isinstance(ts, float) or isinstance(ts, int):
|
|
193
|
+
res = ts
|
|
194
|
+
elif isinstance(ts, str) and (Converter.is_float(ts) or Converter.is_integer(ts)):
|
|
195
|
+
res = Converter.to_float(ts)
|
|
196
|
+
else:
|
|
197
|
+
raise TechnicalException(f"Timestamp must be a float|int or a string containing a float|int (obtained type: {Typing.get_object_class_fullname(ts)})")
|
|
198
|
+
return res
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
### Conversions to/from timestamp
|
|
202
|
+
|
|
179
203
|
@classmethod
|
|
180
204
|
def datetime_to_timestamp(cls, dt, epoch=None):
|
|
205
|
+
dt = cls._get_datetime(dt)
|
|
181
206
|
epoch = cls._get_epoch_datetime(epoch)
|
|
182
207
|
|
|
183
208
|
if epoch is None:
|
|
@@ -190,16 +215,9 @@ class DateTime(object):
|
|
|
190
215
|
"""
|
|
191
216
|
Convert timestamp to UTC datetime
|
|
192
217
|
"""
|
|
218
|
+
ts = cls._get_timestamp(ts)
|
|
193
219
|
epoch = cls._get_epoch_datetime(epoch)
|
|
194
220
|
|
|
195
|
-
if isinstance(ts, str):
|
|
196
|
-
if Converter.is_float(ts) or Converter.is_integer(ts):
|
|
197
|
-
ts = Converter.to_float(ts)
|
|
198
|
-
else:
|
|
199
|
-
raise FunctionalException(f"Timestamp is a string that doesn't contain a float|int (obtained timestamp: '{ts}')")
|
|
200
|
-
if not isinstance(ts, float) and not isinstance(ts, int):
|
|
201
|
-
raise TechnicalException(f"Timestamp must be a float|int or a string containing a float|int (obtained type: {Typing.get_object_class_fullname(ts)})")
|
|
202
|
-
|
|
203
221
|
res = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
|
|
204
222
|
if epoch is not None:
|
|
205
223
|
res = epoch + (res - EPOCH_1970)
|