atomicshop 2.14.3__py3-none-any.whl → 2.14.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/datetimes.py +3 -2
- atomicshop/etws/providers.py +18 -2
- atomicshop/etws/sessions.py +1 -1
- atomicshop/etws/trace.py +0 -1
- atomicshop/file_io/csvs.py +1 -1
- atomicshop/mitm/initialize_engines.py +8 -2
- atomicshop/mitm/initialize_mitm_server.py +24 -7
- atomicshop/mitm/statistic_analyzer.py +376 -3
- atomicshop/wrappers/ctyping/etw_winapi/const.py +130 -20
- atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +141 -5
- atomicshop/wrappers/loggingw/formatters.py +66 -40
- atomicshop/wrappers/loggingw/handlers.py +51 -11
- atomicshop/wrappers/loggingw/loggingw.py +180 -167
- atomicshop/wrappers/loggingw/reading.py +20 -19
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.5.dist-info}/METADATA +1 -1
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.5.dist-info}/RECORD +20 -20
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.5.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.5.dist-info}/WHEEL +0 -0
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.5.dist-info}/top_level.txt +0 -0
|
@@ -1,80 +1,34 @@
|
|
|
1
|
-
import os
|
|
2
1
|
import logging
|
|
2
|
+
import os
|
|
3
3
|
import queue
|
|
4
|
+
from typing import Literal, Union
|
|
4
5
|
|
|
5
6
|
from . import loggers, handlers, formatters
|
|
6
7
|
|
|
7
8
|
|
|
8
|
-
def
|
|
9
|
-
"""
|
|
10
|
-
Function to get a logger and set logging level.
|
|
11
|
-
|
|
12
|
-
:param logger_name: Name of the logger.
|
|
13
|
-
:param logging_level: 'int' or 'str', Logging level to set to the logger.
|
|
14
|
-
None: if None, the logger level will not be set.
|
|
15
|
-
:return: Logger.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
# Get the logger.
|
|
19
|
-
logger: logging.Logger = loggers.get_logger(logger_name)
|
|
20
|
-
# Set the logger level if it is not None.
|
|
21
|
-
if logging_level:
|
|
22
|
-
loggers.set_logging_level(logger, logging_level)
|
|
23
|
-
|
|
24
|
-
return logger
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def get_logger_with_stream_handler(
|
|
28
|
-
logger_name: str, logging_level="DEBUG",
|
|
29
|
-
formatter: str = "%(levelname)s | %(threadName)s | %(name)s | %(message)s"
|
|
30
|
-
) -> logging.Logger:
|
|
31
|
-
"""
|
|
32
|
-
Function to get a logger and add StreamHandler to it.
|
|
33
|
-
|
|
34
|
-
:param logger_name: Name of the logger.
|
|
35
|
-
:param logging_level: 'int' or 'str', Logging level to set to the logger.
|
|
36
|
-
None: if None, the logger level will not be set.
|
|
37
|
-
:param formatter: Formatter to use for StreamHandler. It is template of how a message will look like.
|
|
38
|
-
:return: Logger.
|
|
39
|
-
"""
|
|
40
|
-
|
|
41
|
-
# Get the logger.
|
|
42
|
-
logger: logging.Logger = loggers.get_logger(logger_name)
|
|
43
|
-
# Set the logger level if it is not None.
|
|
44
|
-
if logging_level:
|
|
45
|
-
loggers.set_logging_level(logger, logging_level)
|
|
46
|
-
# Add StreamHandler to the logger.
|
|
47
|
-
add_stream_handler(logger, logging_level, formatter)
|
|
48
|
-
|
|
49
|
-
return logger
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def get_logger_with_timedfilehandler(
|
|
9
|
+
def get_complex_logger(
|
|
53
10
|
logger_name: str,
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
formatter_message_only, disable_duplicate_ms, when, interval, delay, encoding
|
|
63
|
-
)
|
|
64
|
-
|
|
65
|
-
return logger
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def get_logger_with_stream_handler_and_timedfilehandler(
|
|
69
|
-
logger_name: str,
|
|
70
|
-
directory_path,
|
|
71
|
-
file_name: str = None,
|
|
72
|
-
file_extension: str = '.txt',
|
|
11
|
+
file_path: str = None,
|
|
12
|
+
directory_path: str = None,
|
|
13
|
+
add_stream: bool = False,
|
|
14
|
+
add_timedfile: bool = False,
|
|
15
|
+
file_type: Literal[
|
|
16
|
+
'txt',
|
|
17
|
+
'csv',
|
|
18
|
+
'json'] = 'txt',
|
|
73
19
|
logging_level="DEBUG",
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
20
|
+
formatter_streamhandler: Union[
|
|
21
|
+
Literal[
|
|
22
|
+
'MESSAGE',
|
|
23
|
+
'DEFAULT'],
|
|
24
|
+
None] = None,
|
|
25
|
+
formatter_filehandler: Union[
|
|
26
|
+
Literal[
|
|
27
|
+
'MESSAGE',
|
|
28
|
+
'DEFAULT',],
|
|
29
|
+
None] = None,
|
|
30
|
+
formatter_streamhandler_use_nanoseconds: bool = True,
|
|
31
|
+
formatter_filehandler_use_nanoseconds: bool = True,
|
|
78
32
|
when: str = "midnight",
|
|
79
33
|
interval: int = 1,
|
|
80
34
|
delay: bool = True,
|
|
@@ -85,26 +39,35 @@ def get_logger_with_stream_handler_and_timedfilehandler(
|
|
|
85
39
|
Function to get a logger and add StreamHandler and TimedRotatingFileHandler to it.
|
|
86
40
|
|
|
87
41
|
:param logger_name: Name of the logger.
|
|
88
|
-
:param
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
:param
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
42
|
+
:param file_path: full path to the log file. If you don't want to use the file, set it to None.
|
|
43
|
+
You can set the directory_path only and then the 'logger_name' will be used as the file name with the
|
|
44
|
+
'file_type' as the file extension.
|
|
45
|
+
:param directory_path: full path to the directory where the log file will be saved.
|
|
46
|
+
:param add_stream: bool, If set to True, StreamHandler will be added to the logger.
|
|
47
|
+
:param add_timedfile: bool, If set to True, TimedRotatingFileHandler will be added to the logger.
|
|
48
|
+
:param file_type: string, file type of the log file. Default is 'txt'.
|
|
49
|
+
'txt': Text file.
|
|
50
|
+
'csv': CSV file.
|
|
51
|
+
'json': JSON file.
|
|
95
52
|
:param logging_level: str or int, Logging level for the handler, that will use the logger while initiated.
|
|
53
|
+
:param formatter_streamhandler: string, Formatter to use for StreamHandler. It is template of how a message will
|
|
54
|
+
look like.
|
|
55
|
+
None: No formatter will be used.
|
|
56
|
+
'DEFAULT': Default formatter will be used:
|
|
57
|
+
"%(levelname)s | %(threadName)s | %(name)s | %(message)s"
|
|
58
|
+
'MESSAGE': Formatter will be used only for the 'message' part.
|
|
96
59
|
:param formatter_filehandler: string, Formatter to use for handler. It is template of how a message will look like.
|
|
97
60
|
None: No formatter will be used.
|
|
98
|
-
'
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
61
|
+
'DEFAULT': Default formatter will be used for each file extension:
|
|
62
|
+
txt: "%(asctime)s | %(levelname)s | %(threadName)s | %(name)s | %(message)s"
|
|
63
|
+
csv: "%(asctime)s,%(levelname)s,%(threadName)s,%(name)s,%(message)s"
|
|
64
|
+
json: '{"time": "%(asctime)s", "level": "%(levelname)s", "thread": "%(threadName)s",
|
|
102
65
|
"logger": "%(name)s", "message": "%(message)s"}'
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
:param
|
|
107
|
-
'asctime' element.
|
|
66
|
+
'MESSAGE': Formatter will be used only for the 'message' part.
|
|
67
|
+
:param formatter_streamhandler_use_nanoseconds: bool, If set to True, the nanoseconds will be used
|
|
68
|
+
in the formatter in case you provide 'asctime' element.
|
|
69
|
+
:param formatter_filehandler_use_nanoseconds: bool, If set to True, the nanoseconds will be used
|
|
70
|
+
in the formatter in case you provide 'asctime' element.
|
|
108
71
|
:param when: string, When to rotate the log file. Default is 'midnight'.
|
|
109
72
|
[when="midnight"] is set to rotate the filename at midnight. This means that the current file name will be
|
|
110
73
|
added Yesterday's date to the end of the file and today's file will continue to write at the same
|
|
@@ -119,6 +82,7 @@ def get_logger_with_stream_handler_and_timedfilehandler(
|
|
|
119
82
|
:param encoding: string, Encoding to use for the log file. Default is None.
|
|
120
83
|
:param header: string, Header to write to the log file.
|
|
121
84
|
Example: "time,host,error"
|
|
85
|
+
Useful for 'csv' file type format.
|
|
122
86
|
|
|
123
87
|
:return: Logger.
|
|
124
88
|
|
|
@@ -130,11 +94,41 @@ def get_logger_with_stream_handler_and_timedfilehandler(
|
|
|
130
94
|
|
|
131
95
|
def main():
|
|
132
96
|
header: str = "time,host,error"
|
|
133
|
-
|
|
97
|
+
output_log_file: str = "D:\\logs\\log_file.csv"
|
|
98
|
+
|
|
99
|
+
error_logger = loggingw.get_complex_logger(
|
|
100
|
+
logger_name=f'{self.__class__.__name__}_CSV',
|
|
101
|
+
file_path=output_log_file,
|
|
102
|
+
add_timedfile=True,
|
|
103
|
+
file_type='csv',
|
|
104
|
+
formatter_filehandler='MESSAGE',
|
|
105
|
+
header=header
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
error_logger.info(error_message)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
main()
|
|
113
|
+
|
|
114
|
+
------------------------------
|
|
115
|
+
|
|
116
|
+
Example to use StreamHandler to output to console and TimedRotatingFileHandler to write to file:
|
|
117
|
+
from atomicshop.wrappers.loggingw import loggingw
|
|
118
|
+
|
|
134
119
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
120
|
+
def main():
|
|
121
|
+
header: str = "time,host,error"
|
|
122
|
+
output_log_file: str = "D:\\logs\\log_file.txt"
|
|
123
|
+
|
|
124
|
+
error_logger = loggingw.get_complex_logger(
|
|
125
|
+
logger_name=f'{self.__class__.__name__}',
|
|
126
|
+
file_path=output_log_file,
|
|
127
|
+
add_stream=True,
|
|
128
|
+
add_timedfile=True,
|
|
129
|
+
file_type='txt',
|
|
130
|
+
formatter_streamhandler='DEFAULT',
|
|
131
|
+
formatter_filehandler='DEFAULT'
|
|
138
132
|
)
|
|
139
133
|
|
|
140
134
|
error_logger.info(f"{datetime.now()},host1,/path/to/file,error message")
|
|
@@ -143,32 +137,93 @@ def get_logger_with_stream_handler_and_timedfilehandler(
|
|
|
143
137
|
if __name__ == "__main__":
|
|
144
138
|
main()
|
|
145
139
|
"""
|
|
140
|
+
|
|
141
|
+
if not directory_path and not file_path:
|
|
142
|
+
raise ValueError("You need to provide 'directory_path' or 'file_path'.")
|
|
143
|
+
if directory_path and file_path:
|
|
144
|
+
raise ValueError("You can't provide both 'directory_path' and 'file_path'.")
|
|
145
|
+
|
|
146
|
+
if directory_path:
|
|
147
|
+
if directory_path.endswith(os.sep):
|
|
148
|
+
directory_path = directory_path[:-1]
|
|
149
|
+
|
|
150
|
+
file_path = f"{directory_path}{os.sep}{logger_name}.{file_type}"
|
|
151
|
+
|
|
146
152
|
logger = get_logger_with_level(logger_name, logging_level)
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
153
|
+
|
|
154
|
+
if add_stream:
|
|
155
|
+
add_stream_handler(
|
|
156
|
+
logger=logger, logging_level=logging_level, formatter=formatter_streamhandler,
|
|
157
|
+
formatter_use_nanoseconds=formatter_streamhandler_use_nanoseconds)
|
|
158
|
+
|
|
159
|
+
if add_timedfile:
|
|
160
|
+
add_timedfilehandler_with_queuehandler(
|
|
161
|
+
logger=logger, file_path=file_path, logging_level=logging_level, formatter=formatter_filehandler,
|
|
162
|
+
formatter_use_nanoseconds=formatter_filehandler_use_nanoseconds, file_type=file_type,
|
|
163
|
+
when=when, interval=interval, delay=delay, encoding=encoding, header=header)
|
|
164
|
+
|
|
165
|
+
return logger
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def get_logger_with_level(
|
|
169
|
+
logger_name: str,
|
|
170
|
+
logging_level="DEBUG"
|
|
171
|
+
) -> logging.Logger:
|
|
172
|
+
"""
|
|
173
|
+
Function to get a logger and set logging level.
|
|
174
|
+
|
|
175
|
+
:param logger_name: Name of the logger.
|
|
176
|
+
:param logging_level: 'int' or 'str', Logging level to set to the logger.
|
|
177
|
+
None: if None, the logger level will not be set.
|
|
178
|
+
:return: Logger.
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
# Get the logger.
|
|
182
|
+
logger: logging.Logger = loggers.get_logger(logger_name)
|
|
183
|
+
# Set the logger level if it is not None.
|
|
184
|
+
if logging_level:
|
|
185
|
+
loggers.set_logging_level(logger, logging_level)
|
|
152
186
|
|
|
153
187
|
return logger
|
|
154
188
|
|
|
155
189
|
|
|
190
|
+
def _process_formatter_attribute(
|
|
191
|
+
formatter: Union[
|
|
192
|
+
Literal['DEFAULT', 'MESSAGE'],
|
|
193
|
+
None],
|
|
194
|
+
file_type: Union[
|
|
195
|
+
Literal['txt', 'csv', 'json'],
|
|
196
|
+
None] = None
|
|
197
|
+
):
|
|
198
|
+
"""
|
|
199
|
+
Function to process the formatter attribute.
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
if formatter == 'DEFAULT' and file_type is None:
|
|
203
|
+
return formatters.DEFAULT_STREAM_FORMATTER
|
|
204
|
+
elif formatter == 'DEFAULT' and file_type == 'txt':
|
|
205
|
+
return formatters.DEFAULT_FORMATTER_TXT_FILE
|
|
206
|
+
elif formatter == 'DEFAULT' and file_type == 'csv':
|
|
207
|
+
return formatters.DEFAULT_FORMATTER_CSV_FILE
|
|
208
|
+
elif formatter == 'DEFAULT' and file_type == 'json':
|
|
209
|
+
return formatters.DEFAULT_MESSAGE_FORMATTER
|
|
210
|
+
elif formatter == 'MESSAGE':
|
|
211
|
+
return formatters.DEFAULT_MESSAGE_FORMATTER
|
|
212
|
+
else:
|
|
213
|
+
return formatter
|
|
214
|
+
|
|
215
|
+
|
|
156
216
|
def add_stream_handler(
|
|
157
|
-
logger: logging.Logger,
|
|
158
|
-
|
|
159
|
-
|
|
217
|
+
logger: logging.Logger,
|
|
218
|
+
logging_level: str = "DEBUG",
|
|
219
|
+
formatter: Union[
|
|
220
|
+
Literal['DEFAULT', 'MESSAGE'],
|
|
221
|
+
None] = None,
|
|
222
|
+
formatter_use_nanoseconds: bool = False
|
|
160
223
|
):
|
|
161
224
|
"""
|
|
162
225
|
Function to add StreamHandler to logger.
|
|
163
226
|
Stream formatter will output messages to the console.
|
|
164
|
-
|
|
165
|
-
:param logger: Logger to add the handler to.
|
|
166
|
-
:param logging_level: Logging level for the handler, that will use the logger while initiated.
|
|
167
|
-
:param formatter: Formatter to use for StreamHandler. It is template of how a message will look like.
|
|
168
|
-
None: No formatter will be used.
|
|
169
|
-
'default': Default formatter will be used:
|
|
170
|
-
"%(levelname)s | %(threadName)s | %(name)s | %(message)s"
|
|
171
|
-
:param formatter_message_only: bool, If set to True, formatter will be used only for the 'message' part.
|
|
172
227
|
"""
|
|
173
228
|
|
|
174
229
|
# Getting the StreamHandler.
|
|
@@ -177,12 +232,12 @@ def add_stream_handler(
|
|
|
177
232
|
loggers.set_logging_level(stream_handler, logging_level)
|
|
178
233
|
|
|
179
234
|
# If formatter_message_only is set to True, then formatter will be used only for the 'message' part.
|
|
180
|
-
|
|
181
|
-
formatter = "%(message)s"
|
|
235
|
+
formatter = _process_formatter_attribute(formatter)
|
|
182
236
|
|
|
183
237
|
# If formatter was provided, then it will be used.
|
|
184
238
|
if formatter:
|
|
185
|
-
logging_formatter = formatters.get_logging_formatter_from_string(
|
|
239
|
+
logging_formatter = formatters.get_logging_formatter_from_string(
|
|
240
|
+
formatter=formatter, use_nanoseconds=formatter_use_nanoseconds)
|
|
186
241
|
handlers.set_formatter(stream_handler, logging_formatter)
|
|
187
242
|
|
|
188
243
|
# Adding the handler to the main logger
|
|
@@ -194,13 +249,16 @@ def add_stream_handler(
|
|
|
194
249
|
|
|
195
250
|
def add_timedfilehandler_with_queuehandler(
|
|
196
251
|
logger: logging.Logger,
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
252
|
+
file_path: str,
|
|
253
|
+
file_type: Literal[
|
|
254
|
+
'txt',
|
|
255
|
+
'csv',
|
|
256
|
+
'json'] = 'txt',
|
|
200
257
|
logging_level="DEBUG",
|
|
201
|
-
formatter
|
|
202
|
-
|
|
203
|
-
|
|
258
|
+
formatter: Union[
|
|
259
|
+
Literal['DEFAULT', 'MESSAGE'],
|
|
260
|
+
None] = None,
|
|
261
|
+
formatter_use_nanoseconds: bool = False,
|
|
204
262
|
when: str = 'midnight',
|
|
205
263
|
interval: int = 1,
|
|
206
264
|
delay: bool = True,
|
|
@@ -211,45 +269,11 @@ def add_timedfilehandler_with_queuehandler(
|
|
|
211
269
|
Function to add TimedRotatingFileHandler and QueueHandler to logger.
|
|
212
270
|
TimedRotatingFileHandler will output messages to the file through QueueHandler.
|
|
213
271
|
This is needed, since TimedRotatingFileHandler is not thread-safe, though official docs say it is.
|
|
214
|
-
|
|
215
|
-
:param logger: Logger to add the handler to.
|
|
216
|
-
:param directory_path: string, Path to the directory where the log file will be created.
|
|
217
|
-
:param file_name_no_extension: string, Name of the log file without file extension, since we add it through
|
|
218
|
-
separate argument. If not provided, logger name will be used.
|
|
219
|
-
:param file_extension: string, Extension of the log file. Default is '.txt'.
|
|
220
|
-
:param logging_level: str or int, Logging level for the handler, that will use the logger while initiated.
|
|
221
|
-
:param formatter: string, Formatter to use for handler. It is template of how a message will look like.
|
|
222
|
-
None: No formatter will be used.
|
|
223
|
-
'default': Default formatter will be used for each file extension:
|
|
224
|
-
.txt: "%(asctime)s | %(levelname)s | %(threadName)s | %(name)s | %(message)s"
|
|
225
|
-
.csv: "%(asctime)s,%(levelname)s,%(threadName)s,%(name)s,%(message)s"
|
|
226
|
-
.json: '{"time": "%(asctime)s", "level": "%(levelname)s", "thread": "%(threadName)s",
|
|
227
|
-
"logger": "%(name)s", "message": "%(message)s"}'
|
|
228
|
-
:param formatter_message_only: bool, If set to True, formatter will be used only for the 'message' part.
|
|
229
|
-
:param disable_duplicate_ms: bool, If set to True, duplicate milliseconds will be removed from formatter
|
|
230
|
-
'asctime' element.
|
|
231
|
-
:param when: string, When to rotate the log file. Default is 'midnight'.
|
|
232
|
-
[when="midnight"] is set to rotate the filename at midnight. This means that the current file name will be
|
|
233
|
-
added Yesterday's date to the end of the file and today's file will continue to write at the same
|
|
234
|
-
filename. Also, if the script finished working on 25.11.2021, the name of the log file will be "test.log"
|
|
235
|
-
If you run the script again on 28.11.2021, the logging module will take the last modification date of
|
|
236
|
-
the file "test.log" and assign a date to it: test.log.2021_11_25
|
|
237
|
-
The log filename of 28.11.2021 will be called "test.log" again.
|
|
238
|
-
:param interval: int, Interval to rotate the log file. Default is 1.
|
|
239
|
-
If 'when="midnight"' and 'interval=1', then the log file will be rotated every midnight.
|
|
240
|
-
If 'when="midnight"' and 'interval=2', then the log file will be rotated every 2nd midnights.
|
|
241
|
-
:param delay: bool, If set to True, the log file will be created only if there's something to write.
|
|
242
|
-
:param encoding: string, Encoding to use for the log file. Default is None.
|
|
243
|
-
:param header: string, Header to write to the log file.
|
|
244
|
-
Example: "time,host,error"
|
|
245
272
|
"""
|
|
246
273
|
|
|
247
274
|
# If file name wasn't provided we will use the logger name instead.
|
|
248
|
-
if not file_name_no_extension:
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
# Set log file path.
|
|
252
|
-
log_file_path = f'{directory_path}{os.sep}{file_name_no_extension}{file_extension}'
|
|
275
|
+
# if not file_name_no_extension:
|
|
276
|
+
# file_name_no_extension = logger.name
|
|
253
277
|
|
|
254
278
|
# Setting the TimedRotatingFileHandler, without adding it to the logger.
|
|
255
279
|
# It will be added to the QueueListener, which will use the TimedRotatingFileHandler to write logs.
|
|
@@ -259,41 +283,30 @@ def add_timedfilehandler_with_queuehandler(
|
|
|
259
283
|
# Creating file handler with log filename. At this stage the log file is created and locked by the handler,
|
|
260
284
|
# Unless we use "delay=True" to tell the class to write the file only if there's something to write.
|
|
261
285
|
|
|
262
|
-
if
|
|
286
|
+
if file_type == "csv":
|
|
263
287
|
# If file extension is CSV, we'll set the header to the file.
|
|
264
288
|
# This is needed since the CSV file will be rotated, and we'll need to set the header each time.
|
|
265
289
|
# We'll use the custom TimedRotatingFileHandlerWithHeader class.
|
|
266
290
|
file_handler = handlers.get_timed_rotating_file_handler_with_header(
|
|
267
|
-
|
|
291
|
+
file_path, when=when, interval=interval, delay=delay, encoding=encoding, header=header)
|
|
268
292
|
else:
|
|
269
293
|
file_handler = handlers.get_timed_rotating_file_handler(
|
|
270
|
-
|
|
294
|
+
file_path, when=when, interval=interval, delay=delay, encoding=encoding)
|
|
271
295
|
|
|
272
296
|
loggers.set_logging_level(file_handler, logging_level)
|
|
273
297
|
|
|
274
|
-
|
|
275
|
-
# Create file formatter based on extension
|
|
276
|
-
if file_extension == ".txt":
|
|
277
|
-
formatter = formatters.DEFAULT_FORMATTER_TXT_FILE
|
|
278
|
-
elif file_extension == ".csv":
|
|
279
|
-
formatter = formatters.DEFAULT_FORMATTER_CSV_FILE
|
|
280
|
-
elif file_extension == ".json":
|
|
281
|
-
formatter = "%(message)s"
|
|
282
|
-
|
|
283
|
-
# If 'formatter_message_only' is set to 'True', we'll use the formatter only for the message part.
|
|
284
|
-
if formatter_message_only:
|
|
285
|
-
formatter = "%(message)s"
|
|
298
|
+
formatter = _process_formatter_attribute(formatter, file_type=file_type)
|
|
286
299
|
|
|
287
300
|
# If formatter was passed to the function we'll add it to handler.
|
|
288
301
|
if formatter:
|
|
289
302
|
# Convert string to Formatter object. Moved to newer styling of python 3: style='{'
|
|
290
303
|
logging_formatter = formatters.get_logging_formatter_from_string(
|
|
291
|
-
formatter,
|
|
304
|
+
formatter=formatter, use_nanoseconds=formatter_use_nanoseconds)
|
|
292
305
|
# Setting the formatter in file handler.
|
|
293
306
|
handlers.set_formatter(file_handler, logging_formatter)
|
|
294
307
|
|
|
295
308
|
# This function will change the suffix behavior of the rotated file name.
|
|
296
|
-
handlers.change_rotated_filename(file_handler
|
|
309
|
+
handlers.change_rotated_filename(file_handler)
|
|
297
310
|
|
|
298
311
|
queue_handler = start_queue_listener_for_file_handler_and_get_queue_handler(file_handler)
|
|
299
312
|
loggers.set_logging_level(queue_handler, logging_level)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from typing import Literal, Union
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
import datetime
|
|
4
5
|
|
|
5
6
|
from ... import filesystem, datetimes
|
|
6
7
|
from ...file_io import csvs
|
|
@@ -11,7 +12,6 @@ def get_logs_paths(
|
|
|
11
12
|
log_file_path: str = None,
|
|
12
13
|
file_name_pattern: str = '*.*',
|
|
13
14
|
date_pattern: str = None,
|
|
14
|
-
log_type: Literal['csv'] = 'csv',
|
|
15
15
|
latest_only: bool = False,
|
|
16
16
|
previous_day_only: bool = False
|
|
17
17
|
):
|
|
@@ -36,7 +36,6 @@ def get_logs_paths(
|
|
|
36
36
|
:param date_pattern: Pattern to match the date in the log file name.
|
|
37
37
|
If specified, the function will get the log file by the date pattern.
|
|
38
38
|
If not specified, the function will get the file date by file last modified time.
|
|
39
|
-
:param log_type: Type of log to get.
|
|
40
39
|
:param latest_only: Boolean, if True, only the latest log file path will be returned.
|
|
41
40
|
:param previous_day_only: Boolean, if True, only the log file path from the previous day will be returned.
|
|
42
41
|
"""
|
|
@@ -46,9 +45,6 @@ def get_logs_paths(
|
|
|
46
45
|
elif log_files_directory_path and log_file_path:
|
|
47
46
|
raise ValueError('Both "log_files_directory_path" and "log_file_path" cannot be specified at the same time.')
|
|
48
47
|
|
|
49
|
-
if log_type != 'csv':
|
|
50
|
-
raise ValueError('Only "csv" log type is supported.')
|
|
51
|
-
|
|
52
48
|
if latest_only and previous_day_only:
|
|
53
49
|
raise ValueError('Both "latest_only" and "previous_day_only" cannot be True at the same time.')
|
|
54
50
|
|
|
@@ -75,16 +71,22 @@ def get_logs_paths(
|
|
|
75
71
|
for file_index, single_file in enumerate(logs_files):
|
|
76
72
|
# Get file name from current loop file path.
|
|
77
73
|
current_file_name: str = Path(single_file['file_path']).name
|
|
74
|
+
logs_files[file_index]['file_name'] = current_file_name
|
|
75
|
+
|
|
78
76
|
# Get the datetime object from the file name by the date pattern.
|
|
79
77
|
try:
|
|
80
|
-
datetime_object = datetimes.get_datetime_from_complex_string_by_pattern(
|
|
78
|
+
datetime_object, date_string, timestamp_float = datetimes.get_datetime_from_complex_string_by_pattern(
|
|
81
79
|
current_file_name, date_pattern)
|
|
82
|
-
timestamp_float = datetime_object.timestamp()
|
|
83
80
|
# ValueError will be raised if the date pattern does not match the file name.
|
|
84
81
|
except ValueError:
|
|
85
82
|
timestamp_float = 0
|
|
83
|
+
datetime_object = None
|
|
84
|
+
date_string = None
|
|
85
|
+
|
|
86
86
|
# Update the last modified time to the dictionary.
|
|
87
87
|
logs_files[file_index]['last_modified'] = timestamp_float
|
|
88
|
+
logs_files[file_index]['datetime'] = datetime_object
|
|
89
|
+
logs_files[file_index]['date_string'] = date_string
|
|
88
90
|
|
|
89
91
|
if timestamp_float > latest_timestamp:
|
|
90
92
|
latest_timestamp = timestamp_float
|
|
@@ -95,6 +97,8 @@ def get_logs_paths(
|
|
|
95
97
|
if single_file['last_modified'] == 0:
|
|
96
98
|
latest_timestamp += 86400
|
|
97
99
|
logs_files[file_index]['last_modified'] = latest_timestamp
|
|
100
|
+
logs_files[file_index]['datetime'] = datetime.datetime.fromtimestamp(latest_timestamp)
|
|
101
|
+
logs_files[file_index]['date_string'] = logs_files[file_index]['datetime'].strftime(date_pattern)
|
|
98
102
|
break
|
|
99
103
|
|
|
100
104
|
# Sort the files by the last modified time.
|
|
@@ -117,7 +121,7 @@ def get_logs_paths(
|
|
|
117
121
|
return logs_files
|
|
118
122
|
|
|
119
123
|
|
|
120
|
-
def
|
|
124
|
+
def get_all_log_files_into_list(
|
|
121
125
|
log_files_directory_path: str = None,
|
|
122
126
|
log_file_path: str = None,
|
|
123
127
|
file_name_pattern: str = '*.*',
|
|
@@ -127,9 +131,10 @@ def get_logs(
|
|
|
127
131
|
remove_logs: bool = False,
|
|
128
132
|
move_to_path: str = None,
|
|
129
133
|
print_kwargs: dict = None
|
|
130
|
-
):
|
|
134
|
+
) -> list:
|
|
131
135
|
"""
|
|
132
|
-
This function gets the logs from the log files. Supports rotating files to get the logs by time.
|
|
136
|
+
This function gets the logs contents from the log files. Supports rotating files to get the logs by time.
|
|
137
|
+
All the contents will be merged into one list.
|
|
133
138
|
|
|
134
139
|
:param log_files_directory_path: Path to the log files. Check the 'get_logs_paths' function for more details.
|
|
135
140
|
:param log_file_path: Path to the log file. Check the 'get_logs_paths' function for more details.
|
|
@@ -144,8 +149,9 @@ def get_logs(
|
|
|
144
149
|
'all' - Each CSV file has a header. Get the header from each file.
|
|
145
150
|
:param remove_logs: Boolean, if True, the logs will be removed after getting them.
|
|
146
151
|
:param move_to_path: Path to move the logs to.
|
|
147
|
-
|
|
148
152
|
:param print_kwargs: Keyword arguments dict for 'print_api' function.
|
|
153
|
+
|
|
154
|
+
:return: List of dictionaries with the logs content.
|
|
149
155
|
"""
|
|
150
156
|
|
|
151
157
|
if not print_kwargs:
|
|
@@ -162,8 +168,7 @@ def get_logs(
|
|
|
162
168
|
log_files_directory_path=log_files_directory_path,
|
|
163
169
|
log_file_path=log_file_path,
|
|
164
170
|
file_name_pattern=file_name_pattern,
|
|
165
|
-
date_pattern=date_pattern
|
|
166
|
-
log_type=log_type)
|
|
171
|
+
date_pattern=date_pattern)
|
|
167
172
|
|
|
168
173
|
# Read all the logs.
|
|
169
174
|
logs_content: list = list()
|
|
@@ -294,8 +299,7 @@ class LogReader:
|
|
|
294
299
|
# If the existing logs file count is 0, it means that this is the first check. We need to get the current count.
|
|
295
300
|
if self._existing_logs_file_count == 0:
|
|
296
301
|
self._existing_logs_file_count = len(get_logs_paths(
|
|
297
|
-
log_file_path=self.log_file_path
|
|
298
|
-
log_type='csv'
|
|
302
|
+
log_file_path=self.log_file_path
|
|
299
303
|
))
|
|
300
304
|
|
|
301
305
|
# If the count is still 0, then there are no logs to read.
|
|
@@ -311,7 +315,6 @@ class LogReader:
|
|
|
311
315
|
latest_statistics_file_path_object = get_logs_paths(
|
|
312
316
|
log_file_path=self.log_file_path,
|
|
313
317
|
date_pattern=self.date_pattern,
|
|
314
|
-
log_type='csv',
|
|
315
318
|
latest_only=True
|
|
316
319
|
)
|
|
317
320
|
|
|
@@ -327,7 +330,6 @@ class LogReader:
|
|
|
327
330
|
previous_day_statistics_file_path = get_logs_paths(
|
|
328
331
|
log_file_path=self.log_file_path,
|
|
329
332
|
date_pattern=self.date_pattern,
|
|
330
|
-
log_type='csv',
|
|
331
333
|
previous_day_only=True
|
|
332
334
|
)[0]['file_path']
|
|
333
335
|
# If you get IndexError, it means that there are no previous day logs to read.
|
|
@@ -336,8 +338,7 @@ class LogReader:
|
|
|
336
338
|
|
|
337
339
|
# Count all the rotated files.
|
|
338
340
|
current_log_files_count: int = len(get_logs_paths(
|
|
339
|
-
log_file_path=self.log_file_path
|
|
340
|
-
log_type='csv'
|
|
341
|
+
log_file_path=self.log_file_path
|
|
341
342
|
))
|
|
342
343
|
|
|
343
344
|
# If the count of the log files is greater than the existing logs file count, it means that the rotation
|