atomicshop 2.12.24__py3-none-any.whl → 2.12.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/etw/providers.py +5 -0
- atomicshop/etw/sessions.py +29 -0
- atomicshop/etw/trace.py +16 -1
- atomicshop/etw/trace_dns.py +17 -2
- atomicshop/monitor/checks/dns.py +1 -1
- atomicshop/wrappers/ctyping/etw_winapi/const.py +122 -4
- atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +91 -0
- atomicshop/wrappers/loggingw/reading.py +115 -98
- {atomicshop-2.12.24.dist-info → atomicshop-2.12.26.dist-info}/METADATA +1 -1
- {atomicshop-2.12.24.dist-info → atomicshop-2.12.26.dist-info}/RECORD +14 -13
- {atomicshop-2.12.24.dist-info → atomicshop-2.12.26.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.12.24.dist-info → atomicshop-2.12.26.dist-info}/WHEEL +0 -0
- {atomicshop-2.12.24.dist-info → atomicshop-2.12.26.dist-info}/top_level.txt +0 -0
atomicshop/__init__.py
CHANGED
atomicshop/etw/sessions.py
CHANGED
|
@@ -12,3 +12,32 @@ def stop_and_delete(session_name) -> tuple[bool, int]:
|
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
14
|
return etw_functions.stop_and_delete_etw_session(session_name)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_running_list() -> list[dict]:
|
|
18
|
+
"""
|
|
19
|
+
List all running ETW sessions.
|
|
20
|
+
|
|
21
|
+
:return: A list of strings containing the names of all running ETW sessions.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
return etw_functions.list_etw_sessions()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def is_session_running(session_name: str) -> bool:
|
|
28
|
+
"""
|
|
29
|
+
Check if an ETW session is running.
|
|
30
|
+
|
|
31
|
+
:param session_name: The name of the session to check.
|
|
32
|
+
:return: A boolean indicating if the session is running.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
# Get all running sessions.
|
|
36
|
+
running_sessions = get_running_list()
|
|
37
|
+
|
|
38
|
+
# Check if the session is in the list of running sessions.
|
|
39
|
+
for session in running_sessions:
|
|
40
|
+
if session['session_name'] == session_name:
|
|
41
|
+
return True
|
|
42
|
+
|
|
43
|
+
return False
|
atomicshop/etw/trace.py
CHANGED
|
@@ -5,6 +5,7 @@ import sys
|
|
|
5
5
|
import etw
|
|
6
6
|
|
|
7
7
|
from ..print_api import print_api
|
|
8
|
+
from . import sessions
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class EventTrace(etw.ETW):
|
|
@@ -13,7 +14,8 @@ class EventTrace(etw.ETW):
|
|
|
13
14
|
providers: list,
|
|
14
15
|
event_callback=None,
|
|
15
16
|
event_id_filters: list = None,
|
|
16
|
-
session_name: str = None
|
|
17
|
+
session_name: str = None,
|
|
18
|
+
close_existing_session_name: bool = True
|
|
17
19
|
):
|
|
18
20
|
"""
|
|
19
21
|
:param providers: List of tuples with provider name and provider GUID.
|
|
@@ -23,6 +25,7 @@ class EventTrace(etw.ETW):
|
|
|
23
25
|
:param event_id_filters: List of event IDs that we want to filter. If not provided, all events will be returned.
|
|
24
26
|
The default in the 'etw.ETW' method is 'None'.
|
|
25
27
|
:param session_name: The name of the session to create. If not provided, a UUID will be generated.
|
|
28
|
+
:param close_existing_session_name: Boolean to close existing session names.
|
|
26
29
|
------------------------------------------
|
|
27
30
|
You should stop the ETW tracing when you are done with it.
|
|
28
31
|
'pywintrace' module starts a new session for ETW tracing, and it will not stop the session when the script
|
|
@@ -42,6 +45,7 @@ class EventTrace(etw.ETW):
|
|
|
42
45
|
atexits.run_callable_on_exit_and_signals(EventTrace.stop)
|
|
43
46
|
"""
|
|
44
47
|
self.event_queue = queue.Queue()
|
|
48
|
+
self.close_existing_session_name: bool = close_existing_session_name
|
|
45
49
|
|
|
46
50
|
# If no callback function is provided, we will use the default one, which will put the event in the queue.
|
|
47
51
|
if not event_callback:
|
|
@@ -61,6 +65,17 @@ class EventTrace(etw.ETW):
|
|
|
61
65
|
)
|
|
62
66
|
|
|
63
67
|
def start(self):
|
|
68
|
+
# Check if the session name already exists.
|
|
69
|
+
if sessions.is_session_running(self.session_name):
|
|
70
|
+
print_api(f'ETW Session already running: {self.session_name}', color='yellow')
|
|
71
|
+
|
|
72
|
+
# Close the existing session name.
|
|
73
|
+
if self.close_existing_session_name:
|
|
74
|
+
print_api(f'Closing existing session: {self.session_name}', color='blue')
|
|
75
|
+
sessions.stop_and_delete(self.session_name)
|
|
76
|
+
else:
|
|
77
|
+
print_api(f'Using existing session: {self.session_name}', color='yellow')
|
|
78
|
+
|
|
64
79
|
try:
|
|
65
80
|
super().start()
|
|
66
81
|
except OSError as e:
|
atomicshop/etw/trace_dns.py
CHANGED
|
@@ -7,7 +7,13 @@ from ..print_api import print_api
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class DnsTrace:
|
|
10
|
-
def __init__(
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
enable_process_poller: bool = False,
|
|
13
|
+
attrs: list = None,
|
|
14
|
+
session_name: str = None,
|
|
15
|
+
close_existing_session_name: bool = True
|
|
16
|
+
):
|
|
11
17
|
"""
|
|
12
18
|
DnsTrace class use to trace DNS events from Windows Event Tracing for EventId 3008.
|
|
13
19
|
|
|
@@ -16,6 +22,14 @@ class DnsTrace:
|
|
|
16
22
|
Then DNS events will be enriched with the process name and command line from the process poller.
|
|
17
23
|
:param attrs: List of attributes to return. If None, all attributes will be returned.
|
|
18
24
|
:param session_name: The name of the session to create. If not provided, a UUID will be generated.
|
|
25
|
+
:param close_existing_session_name: Boolean to close existing session names.
|
|
26
|
+
True: if ETW session with 'session_name' exists, you will be notified and the session will be closed.
|
|
27
|
+
Then the new session with this name will be created.
|
|
28
|
+
False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
|
|
29
|
+
created. Instead, the existing session will be used. If there is a buffer from the previous session,
|
|
30
|
+
you will get the events from the buffer.
|
|
31
|
+
|
|
32
|
+
-------------------------------------------------
|
|
19
33
|
|
|
20
34
|
Usage Example:
|
|
21
35
|
from atomicshop.etw import dns_trace
|
|
@@ -36,7 +50,8 @@ class DnsTrace:
|
|
|
36
50
|
providers=[(dns.ETW_DNS_INFO['provider_name'], dns.ETW_DNS_INFO['provider_guid'])],
|
|
37
51
|
# lambda x: self.event_queue.put(x),
|
|
38
52
|
event_id_filters=[dns.ETW_DNS_INFO['event_id']],
|
|
39
|
-
session_name=session_name
|
|
53
|
+
session_name=session_name,
|
|
54
|
+
close_existing_session_name=close_existing_session_name
|
|
40
55
|
)
|
|
41
56
|
|
|
42
57
|
if self.enable_process_poller:
|
atomicshop/monitor/checks/dns.py
CHANGED
|
@@ -30,7 +30,7 @@ class DnsCheck:
|
|
|
30
30
|
self.fetch_engine: DnsTrace = (
|
|
31
31
|
DnsTrace(
|
|
32
32
|
enable_process_poller=True, attrs=['name', 'cmdline', 'domain', 'query_type'],
|
|
33
|
-
session_name=self.etw_session_name)
|
|
33
|
+
session_name=self.etw_session_name, close_existing_session_name=True)
|
|
34
34
|
)
|
|
35
35
|
|
|
36
36
|
if self.settings['alert_always'] and self.settings['alert_about_missing_entries_after_learning']:
|
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
import ctypes
|
|
2
2
|
from ctypes import wintypes
|
|
3
|
+
from ctypes.wintypes import ULONG
|
|
3
4
|
|
|
4
5
|
|
|
5
|
-
# Load the necessary library
|
|
6
|
-
advapi32 = ctypes.WinDLL('advapi32')
|
|
7
|
-
|
|
8
6
|
# Constants
|
|
9
7
|
EVENT_TRACE_CONTROL_STOP = 1
|
|
10
8
|
WNODE_FLAG_TRACED_GUID = 0x00020000
|
|
11
9
|
|
|
10
|
+
MAXIMUM_LOGGERS = 64
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
"""
|
|
14
|
+
wintypes.DWORD = wintypes.ULONG = ctypes.c_ulong: 32-bit unsigned integer
|
|
15
|
+
wintypes.WORD = wintypes.USHORT = ctypes.c_ushort: 16-bit unsigned integer
|
|
16
|
+
wintypes.BYTE = ctypes.c_ubyte: 8-bit unsigned integer
|
|
17
|
+
wintypes.LARGE_INTEGER is a structure (or union in C terms), can represent both signed and unsigned
|
|
18
|
+
64-bit values depending on context.
|
|
19
|
+
ctypes.c_ulonglong is a simple data type representing an unsigned 64-bit integer.
|
|
20
|
+
"""
|
|
21
|
+
|
|
12
22
|
|
|
13
23
|
# Define GUID structure
|
|
14
24
|
class GUID(ctypes.Structure):
|
|
@@ -53,5 +63,113 @@ class EVENT_TRACE_PROPERTIES(ctypes.Structure):
|
|
|
53
63
|
("RealTimeBuffersLost", wintypes.ULONG),
|
|
54
64
|
("LoggerThreadId", wintypes.HANDLE),
|
|
55
65
|
("LogFileNameOffset", wintypes.ULONG),
|
|
56
|
-
("LoggerNameOffset", wintypes.ULONG)
|
|
66
|
+
("LoggerNameOffset", wintypes.ULONG),
|
|
67
|
+
# Allocate space for the names at the end of the structure
|
|
68
|
+
("_LoggerName", wintypes.WCHAR * 1024),
|
|
69
|
+
("_LogFileName", wintypes.WCHAR * 1024)
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# Define the EVENT_TRACE_LOGFILE structure
|
|
74
|
+
class EVENT_TRACE_LOGFILE(ctypes.Structure):
|
|
75
|
+
_fields_ = [
|
|
76
|
+
("LogFileName", wintypes.LPWSTR),
|
|
77
|
+
("LoggerName", wintypes.LPWSTR),
|
|
78
|
+
("CurrentTime", wintypes.LARGE_INTEGER),
|
|
79
|
+
("BuffersRead", wintypes.ULONG),
|
|
80
|
+
("ProcessTraceMode", wintypes.ULONG),
|
|
81
|
+
("EventRecordCallback", wintypes.LPVOID),
|
|
82
|
+
("BufferSize", wintypes.ULONG),
|
|
83
|
+
("Filled", wintypes.ULONG),
|
|
84
|
+
("EventsLost", wintypes.ULONG),
|
|
85
|
+
("BuffersLost", wintypes.ULONG),
|
|
86
|
+
("RealTimeBuffersLost", wintypes.ULONG),
|
|
87
|
+
("LogBuffersLost", wintypes.ULONG),
|
|
88
|
+
("BuffersWritten", wintypes.ULONG),
|
|
89
|
+
("LogFileMode", wintypes.ULONG),
|
|
90
|
+
("IsKernelTrace", wintypes.ULONG),
|
|
91
|
+
("Context", wintypes.ULONG) # Placeholder for context pointer
|
|
57
92
|
]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# Define the EVENT_TRACE_HEADER structure
|
|
96
|
+
class EVENT_TRACE_HEADER(ctypes.Structure):
|
|
97
|
+
_fields_ = [
|
|
98
|
+
("Size", wintypes.USHORT),
|
|
99
|
+
("FieldTypeFlags", wintypes.USHORT),
|
|
100
|
+
("Version", wintypes.USHORT),
|
|
101
|
+
("Class", wintypes.USHORT), # EVENT_TRACE_CLASS
|
|
102
|
+
("Type", ctypes.c_ubyte),
|
|
103
|
+
("Level", ctypes.c_ubyte),
|
|
104
|
+
("Channel", ctypes.c_ubyte),
|
|
105
|
+
("Flags", ctypes.c_ubyte),
|
|
106
|
+
("InstanceId", wintypes.USHORT),
|
|
107
|
+
("ParentInstanceId", wintypes.USHORT),
|
|
108
|
+
("ParentGuid", GUID),
|
|
109
|
+
("Timestamp", wintypes.LARGE_INTEGER),
|
|
110
|
+
("Guid", GUID),
|
|
111
|
+
("ProcessorTime", wintypes.ULONG),
|
|
112
|
+
("ThreadId", wintypes.ULONG),
|
|
113
|
+
("ProcessId", wintypes.ULONG),
|
|
114
|
+
("KernelTime", wintypes.ULONG),
|
|
115
|
+
("UserTime", wintypes.ULONG),
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# Define the EVENT_RECORD structure
|
|
120
|
+
class EVENT_RECORD(ctypes.Structure):
|
|
121
|
+
_fields_ = [
|
|
122
|
+
("EventHeader", EVENT_TRACE_HEADER),
|
|
123
|
+
("BufferContext", wintypes.ULONG),
|
|
124
|
+
("ExtendedDataCount", wintypes.USHORT),
|
|
125
|
+
("UserDataLength", wintypes.USHORT),
|
|
126
|
+
("ExtendedData", wintypes.LPVOID),
|
|
127
|
+
("UserData", wintypes.LPVOID),
|
|
128
|
+
("UserContext", wintypes.LPVOID)
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class PROVIDER_ENUMERATION_INFO(ctypes.Structure):
|
|
133
|
+
_fields_ = [
|
|
134
|
+
("NumberOfProviders", ULONG),
|
|
135
|
+
("Reserved", ULONG),
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class PROVIDER_INFORMATION(ctypes.Structure):
|
|
140
|
+
_fields_ = [
|
|
141
|
+
("ProviderId", ctypes.c_byte * 16),
|
|
142
|
+
("SchemaSource", ULONG),
|
|
143
|
+
("ProviderNameOffset", ULONG),
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# Load the necessary library
|
|
148
|
+
advapi32 = ctypes.WinDLL('advapi32')
|
|
149
|
+
tdh = ctypes.windll.tdh
|
|
150
|
+
|
|
151
|
+
# Define necessary TDH functions
|
|
152
|
+
tdh.TdhEnumerateProviders.argtypes = [ctypes.POINTER(PROVIDER_ENUMERATION_INFO), ctypes.POINTER(ULONG)]
|
|
153
|
+
tdh.TdhEnumerateProviders.restype = ULONG
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# Define the function prototype
|
|
157
|
+
QueryAllTraces = advapi32.QueryAllTracesW
|
|
158
|
+
QueryAllTraces.argtypes = [
|
|
159
|
+
ctypes.POINTER(ctypes.POINTER(EVENT_TRACE_PROPERTIES)),
|
|
160
|
+
wintypes.ULONG,
|
|
161
|
+
ctypes.POINTER(wintypes.ULONG)
|
|
162
|
+
]
|
|
163
|
+
QueryAllTraces.restype = wintypes.ULONG
|
|
164
|
+
|
|
165
|
+
OpenTrace = advapi32.OpenTraceW
|
|
166
|
+
OpenTrace.argtypes = [ctypes.POINTER(EVENT_TRACE_LOGFILE)]
|
|
167
|
+
OpenTrace.restype = wintypes.ULONG
|
|
168
|
+
|
|
169
|
+
ProcessTrace = advapi32.ProcessTrace
|
|
170
|
+
ProcessTrace.argtypes = [ctypes.POINTER(wintypes.ULONG), wintypes.ULONG, wintypes.LARGE_INTEGER, wintypes.LARGE_INTEGER]
|
|
171
|
+
ProcessTrace.restype = wintypes.ULONG
|
|
172
|
+
|
|
173
|
+
CloseTrace = advapi32.CloseTrace
|
|
174
|
+
CloseTrace.argtypes = [wintypes.ULONG]
|
|
175
|
+
CloseTrace.restype = wintypes.ULONG
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import ctypes
|
|
2
|
+
from ctypes.wintypes import ULONG
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
2
5
|
from . import const
|
|
3
6
|
|
|
4
7
|
|
|
@@ -35,3 +38,91 @@ def stop_and_delete_etw_session(session_name) -> tuple[bool, int]:
|
|
|
35
38
|
else:
|
|
36
39
|
# print("ETW session stopped and deleted successfully.")
|
|
37
40
|
return True, status
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def get_all_providers() -> list[tuple[any, uuid.UUID]]:
|
|
44
|
+
"""
|
|
45
|
+
Get all ETW providers available on the system.
|
|
46
|
+
|
|
47
|
+
:return: A list of tuples containing the provider name and GUID.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
providers_info_size = ULONG(0)
|
|
51
|
+
status = const.tdh.TdhEnumerateProviders(None, ctypes.byref(providers_info_size))
|
|
52
|
+
|
|
53
|
+
# Initial allocation
|
|
54
|
+
buffer = (ctypes.c_byte * providers_info_size.value)()
|
|
55
|
+
providers_info = ctypes.cast(buffer, ctypes.POINTER(const.PROVIDER_ENUMERATION_INFO))
|
|
56
|
+
|
|
57
|
+
# Loop to handle resizing
|
|
58
|
+
while True:
|
|
59
|
+
status = const.tdh.TdhEnumerateProviders(providers_info, ctypes.byref(providers_info_size))
|
|
60
|
+
|
|
61
|
+
if status == 0:
|
|
62
|
+
break
|
|
63
|
+
elif status == 0x8007007A: # ERROR_INSUFFICIENT_BUFFER
|
|
64
|
+
buffer = (ctypes.c_byte * providers_info_size.value)()
|
|
65
|
+
providers_info = ctypes.cast(buffer, ctypes.POINTER(const.PROVIDER_ENUMERATION_INFO))
|
|
66
|
+
else:
|
|
67
|
+
raise ctypes.WinError(status)
|
|
68
|
+
|
|
69
|
+
provider_count = providers_info.contents.NumberOfProviders
|
|
70
|
+
provider_array = ctypes.cast(
|
|
71
|
+
ctypes.addressof(providers_info.contents) + ctypes.sizeof(const.PROVIDER_ENUMERATION_INFO),
|
|
72
|
+
ctypes.POINTER(const.PROVIDER_INFORMATION * provider_count))
|
|
73
|
+
|
|
74
|
+
providers = []
|
|
75
|
+
for i in range(provider_count):
|
|
76
|
+
provider = provider_array.contents[i]
|
|
77
|
+
provider_name_offset = provider.ProviderNameOffset
|
|
78
|
+
provider_name_ptr = ctypes.cast(
|
|
79
|
+
ctypes.addressof(providers_info.contents) + provider_name_offset, ctypes.c_wchar_p)
|
|
80
|
+
provider_name = provider_name_ptr.value
|
|
81
|
+
provider_guid = uuid.UUID(bytes_le=bytes(provider.ProviderId))
|
|
82
|
+
providers.append((provider_name, provider_guid))
|
|
83
|
+
|
|
84
|
+
return providers
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def list_etw_sessions() -> list[dict]:
|
|
88
|
+
"""
|
|
89
|
+
List all running ETW sessions.
|
|
90
|
+
|
|
91
|
+
:return: A list of dictionaries containing the names of all running ETW sessions and their log files.
|
|
92
|
+
"""
|
|
93
|
+
# Create an array of EVENT_TRACE_PROPERTIES pointers
|
|
94
|
+
PropertiesArrayType = ctypes.POINTER(const.EVENT_TRACE_PROPERTIES) * const.MAXIMUM_LOGGERS
|
|
95
|
+
properties_array = PropertiesArrayType()
|
|
96
|
+
for i in range(const.MAXIMUM_LOGGERS):
|
|
97
|
+
properties = const.EVENT_TRACE_PROPERTIES()
|
|
98
|
+
properties.Wnode.BufferSize = ctypes.sizeof(const.EVENT_TRACE_PROPERTIES)
|
|
99
|
+
properties_array[i] = ctypes.pointer(properties)
|
|
100
|
+
|
|
101
|
+
# Define the number of loggers variable
|
|
102
|
+
logger_count = ULONG(const.MAXIMUM_LOGGERS)
|
|
103
|
+
|
|
104
|
+
# Call QueryAllTraces
|
|
105
|
+
status = const.QueryAllTraces(properties_array, const.MAXIMUM_LOGGERS, ctypes.byref(logger_count))
|
|
106
|
+
if status != 0:
|
|
107
|
+
raise Exception(f"QueryAllTraces failed, error code: {status}")
|
|
108
|
+
|
|
109
|
+
# Extract session names
|
|
110
|
+
session_list: list = []
|
|
111
|
+
for i in range(logger_count.value):
|
|
112
|
+
logger_name = None
|
|
113
|
+
logfile_path = None
|
|
114
|
+
|
|
115
|
+
properties = properties_array[i].contents
|
|
116
|
+
if properties.LoggerNameOffset != 0:
|
|
117
|
+
logger_name_address = ctypes.addressof(properties) + properties.LoggerNameOffset
|
|
118
|
+
logger_name = ctypes.wstring_at(logger_name_address)
|
|
119
|
+
if properties.LogFileNameOffset != 0:
|
|
120
|
+
logfile_name_address = ctypes.addressof(properties) + properties.LogFileNameOffset
|
|
121
|
+
logfile_path = ctypes.wstring_at(logfile_name_address)
|
|
122
|
+
|
|
123
|
+
session_list.append({
|
|
124
|
+
'session_name': logger_name,
|
|
125
|
+
'log_file': logfile_path
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
return session_list
|
|
@@ -6,10 +6,6 @@ from ... import filesystem, datetimes
|
|
|
6
6
|
from ...file_io import csvs
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
READING_EXISTING_LINES: list = []
|
|
10
|
-
EXISTING_LOGS_FILE_COUNT: int = 0
|
|
11
|
-
|
|
12
|
-
|
|
13
9
|
def get_logs_paths(
|
|
14
10
|
log_files_directory_path: str = None,
|
|
15
11
|
log_file_path: str = None,
|
|
@@ -212,31 +208,10 @@ def get_logs(
|
|
|
212
208
|
return logs_content
|
|
213
209
|
|
|
214
210
|
|
|
215
|
-
|
|
216
|
-
log_file_path: str,
|
|
217
|
-
date_pattern: str = None,
|
|
218
|
-
log_type: Literal['csv'] = 'csv',
|
|
219
|
-
get_previous_file: bool = False,
|
|
220
|
-
header: list = None
|
|
221
|
-
) -> tuple:
|
|
211
|
+
class LogReader:
|
|
222
212
|
"""
|
|
223
|
-
This
|
|
213
|
+
This class gets the latest lines from the log file.
|
|
224
214
|
|
|
225
|
-
:param log_file_path: Path to the log file.
|
|
226
|
-
:param date_pattern: Pattern to match the date in the log file name.
|
|
227
|
-
If specified, the function will get the log file by the date pattern.
|
|
228
|
-
If not specified, the function will get the file date by file last modified time.
|
|
229
|
-
:param log_type: Type of log to get.
|
|
230
|
-
:param get_previous_file: Boolean, if True, the function will get the previous log file.
|
|
231
|
-
For example, your log is set to rotate every Midnight.
|
|
232
|
-
Meaning, once the day will change, the function will get the log file from the previous day in the third entry
|
|
233
|
-
of the return tuple. This happens only once each 24 hours. Not from the time the function was called, but from
|
|
234
|
-
the time the day changed.
|
|
235
|
-
:param header: List of strings that will be the header of the CSV file. Default is 'None'.
|
|
236
|
-
None: the header from the CSV file will be used. The first row of the CSV file will be the header.
|
|
237
|
-
Meaning, that the first line will be skipped and the second line will be the first row of the content.
|
|
238
|
-
List: the list will be used as header.
|
|
239
|
-
All the lines of the CSV file will be considered as content.
|
|
240
215
|
return: List of new lines.
|
|
241
216
|
|
|
242
217
|
Usage:
|
|
@@ -246,14 +221,15 @@ def get_latest_lines(
|
|
|
246
221
|
# The header of the log file will be read from the first iteration of the log file.
|
|
247
222
|
# When the file is rotated, this header will be used to not read the header again.
|
|
248
223
|
header: Union[list, None] = None
|
|
224
|
+
log_reader = reading.LogReader(
|
|
225
|
+
log_file_path='/path/to/log.csv',
|
|
226
|
+
log_type='csv',
|
|
227
|
+
date_pattern='%Y_%m_%d',
|
|
228
|
+
get_previous_file=True,
|
|
229
|
+
header=header
|
|
230
|
+
)
|
|
249
231
|
while True:
|
|
250
|
-
latest_lines, previous_day_24h_lines, header =
|
|
251
|
-
log_file_path='/path/to/log.csv',
|
|
252
|
-
log_type='csv',
|
|
253
|
-
date_pattern='%Y_%m_%d',
|
|
254
|
-
get_previous_file=True,
|
|
255
|
-
header=header
|
|
256
|
-
)
|
|
232
|
+
latest_lines, previous_day_24h_lines, header = log_reader.get_latest_lines(header=header)
|
|
257
233
|
|
|
258
234
|
if latest_lines:
|
|
259
235
|
# Do something with the new lines.
|
|
@@ -262,91 +238,132 @@ def get_latest_lines(
|
|
|
262
238
|
# Do something with the last 24 hours lines. Reminder, this will happen once a day on log rotation.
|
|
263
239
|
|
|
264
240
|
time.sleep(1)
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
def __init__(
|
|
244
|
+
self,
|
|
245
|
+
log_file_path: str,
|
|
246
|
+
date_pattern: str = None,
|
|
247
|
+
log_type: Literal['csv'] = 'csv',
|
|
248
|
+
get_previous_file: bool = False,
|
|
249
|
+
header: list = None
|
|
250
|
+
):
|
|
251
|
+
"""
|
|
252
|
+
:param log_file_path: Path to the log file.
|
|
253
|
+
:param date_pattern: Pattern to match the date in the log file name.
|
|
254
|
+
If specified, the function will get the log file by the date pattern.
|
|
255
|
+
If not specified, the function will get the file date by file last modified time.
|
|
256
|
+
:param log_type: Type of log to get.
|
|
257
|
+
:param get_previous_file: Boolean, if True, the function will get the previous log file.
|
|
258
|
+
For example, your log is set to rotate every Midnight.
|
|
259
|
+
Meaning, once the day will change, the function will get the log file from the previous day in the third entry
|
|
260
|
+
of the return tuple. This happens only once each 24 hours. Not from the time the function was called, but from
|
|
261
|
+
the time the day changed.
|
|
262
|
+
:param header: List of strings that will be the header of the CSV file. Default is 'None'.
|
|
263
|
+
None: the header from the CSV file will be used. The first row of the CSV file will be the header.
|
|
264
|
+
Meaning, that the first line will be skipped and the second line will be the first row of the content.
|
|
265
|
+
List: the list will be used as header.
|
|
266
|
+
All the lines of the CSV file will be considered as content.
|
|
267
|
+
"""
|
|
268
|
+
|
|
269
|
+
self.log_file_path: str = log_file_path
|
|
270
|
+
self.date_pattern: str = date_pattern
|
|
271
|
+
self.log_type: Literal['csv'] = log_type
|
|
272
|
+
self.get_previous_file: bool = get_previous_file
|
|
273
|
+
self.header: list = header
|
|
274
|
+
|
|
275
|
+
self._reading_existing_lines: list = []
|
|
276
|
+
self._existing_logs_file_count: int = 0
|
|
277
|
+
|
|
278
|
+
def _extract_new_lines_only(self, content_lines: list):
|
|
268
279
|
new_lines: list = []
|
|
269
280
|
for row in content_lines:
|
|
270
281
|
# If the row is not in the existing lines, then add it to the new lines.
|
|
271
|
-
if row not in
|
|
282
|
+
if row not in self._reading_existing_lines:
|
|
272
283
|
new_lines.append(row)
|
|
273
284
|
|
|
274
285
|
if new_lines:
|
|
275
|
-
|
|
286
|
+
self._reading_existing_lines.extend(new_lines)
|
|
276
287
|
|
|
277
288
|
return new_lines
|
|
278
289
|
|
|
279
|
-
|
|
290
|
+
def get_latest_lines(self, header: list = None) -> tuple:
|
|
291
|
+
if header:
|
|
292
|
+
self.header = header
|
|
280
293
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
294
|
+
# If the existing logs file count is 0, it means that this is the first check. We need to get the current count.
|
|
295
|
+
if self._existing_logs_file_count == 0:
|
|
296
|
+
self._existing_logs_file_count = len(get_logs_paths(
|
|
297
|
+
log_file_path=self.log_file_path,
|
|
298
|
+
log_type='csv'
|
|
299
|
+
))
|
|
287
300
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
301
|
+
# If the count is still 0, then there are no logs to read.
|
|
302
|
+
if self._existing_logs_file_count == 0:
|
|
303
|
+
return [], [], self.header
|
|
291
304
|
|
|
292
|
-
|
|
293
|
-
|
|
305
|
+
if self.log_type != 'csv':
|
|
306
|
+
raise ValueError('Only "csv" log type is supported.')
|
|
294
307
|
|
|
295
|
-
|
|
308
|
+
previous_file_lines: list = []
|
|
296
309
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
log_type='csv',
|
|
302
|
-
latest_only=True
|
|
303
|
-
)
|
|
304
|
-
|
|
305
|
-
latest_statistics_file_path: str = latest_statistics_file_path_object[0]['file_path']
|
|
306
|
-
|
|
307
|
-
# Get the previous day statistics file path.
|
|
308
|
-
previous_day_statistics_file_path: Union[str, None] = None
|
|
309
|
-
try:
|
|
310
|
-
previous_day_statistics_file_path = get_logs_paths(
|
|
311
|
-
log_file_path=log_file_path,
|
|
312
|
-
date_pattern=date_pattern,
|
|
310
|
+
# Get the latest statistics file path.
|
|
311
|
+
latest_statistics_file_path_object = get_logs_paths(
|
|
312
|
+
log_file_path=self.log_file_path,
|
|
313
|
+
date_pattern=self.date_pattern,
|
|
313
314
|
log_type='csv',
|
|
314
|
-
|
|
315
|
-
)
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
315
|
+
latest_only=True
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# # If there are no logs to read, return empty lists.
|
|
319
|
+
# if not latest_statistics_file_path_object:
|
|
320
|
+
# return [], [], self.header
|
|
321
|
+
|
|
322
|
+
latest_statistics_file_path: str = latest_statistics_file_path_object[0]['file_path']
|
|
323
|
+
|
|
324
|
+
# Get the previous day statistics file path.
|
|
325
|
+
previous_day_statistics_file_path: Union[str, None] = None
|
|
326
|
+
try:
|
|
327
|
+
previous_day_statistics_file_path = get_logs_paths(
|
|
328
|
+
log_file_path=self.log_file_path,
|
|
329
|
+
date_pattern=self.date_pattern,
|
|
330
|
+
log_type='csv',
|
|
331
|
+
previous_day_only=True
|
|
332
|
+
)[0]['file_path']
|
|
333
|
+
# If you get IndexError, it means that there are no previous day logs to read.
|
|
334
|
+
except IndexError:
|
|
335
|
+
pass
|
|
336
|
+
|
|
337
|
+
# Count all the rotated files.
|
|
338
|
+
current_log_files_count: int = len(get_logs_paths(
|
|
339
|
+
log_file_path=self.log_file_path,
|
|
340
|
+
log_type='csv'
|
|
341
|
+
))
|
|
325
342
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
343
|
+
# If the count of the log files is greater than the existing logs file count, it means that the rotation
|
|
344
|
+
# happened. We will read the previous day statistics file.
|
|
345
|
+
new_lines_from_previous_file: list = []
|
|
346
|
+
if current_log_files_count > self._existing_logs_file_count:
|
|
347
|
+
current_lines, self.header = csvs.read_csv_to_list_of_dicts_by_header(
|
|
348
|
+
previous_day_statistics_file_path, header=self.header, stdout=False)
|
|
332
349
|
|
|
333
|
-
|
|
334
|
-
|
|
350
|
+
if self.get_previous_file:
|
|
351
|
+
previous_file_lines = current_lines
|
|
335
352
|
|
|
336
|
-
|
|
353
|
+
self._existing_logs_file_count = current_log_files_count
|
|
337
354
|
|
|
338
|
-
|
|
355
|
+
new_lines_from_previous_file = self._extract_new_lines_only(current_lines)
|
|
339
356
|
|
|
340
|
-
|
|
341
|
-
|
|
357
|
+
# empty the previous file lines, since the file is rotated.
|
|
358
|
+
self._reading_existing_lines.clear()
|
|
342
359
|
|
|
343
|
-
|
|
344
|
-
|
|
360
|
+
current_lines, self.header = csvs.read_csv_to_list_of_dicts_by_header(
|
|
361
|
+
latest_statistics_file_path, header=self.header, stdout=False)
|
|
345
362
|
|
|
346
|
-
|
|
363
|
+
new_lines = self._extract_new_lines_only(current_lines)
|
|
347
364
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
365
|
+
# If we have new lines from the previous file, we will add the new lines from the latest file.
|
|
366
|
+
if new_lines_from_previous_file:
|
|
367
|
+
new_lines = new_lines_from_previous_file + new_lines
|
|
351
368
|
|
|
352
|
-
|
|
369
|
+
return new_lines, previous_file_lines, self.header
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
atomicshop/__init__.py,sha256=
|
|
1
|
+
atomicshop/__init__.py,sha256=Wq0bf4jFBHfU0lsKuJb7kZ5yruxZGg3_luyY2rWwPss,124
|
|
2
2
|
atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
|
|
3
3
|
atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
|
|
4
4
|
atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
|
|
@@ -102,9 +102,10 @@ atomicshop/basics/threads.py,sha256=xvgdDJdmgN0wmmARoZ-H7Kvl1GOcEbvgaeGL4M3Hcx8,
|
|
|
102
102
|
atomicshop/basics/timeit_template.py,sha256=fYLrk-X_dhdVtnPU22tarrhhvlggeW6FdKCXM8zkX68,405
|
|
103
103
|
atomicshop/basics/tracebacks.py,sha256=cNfh_oAwF55kSIdqtv3boHZQIoQI8TajxkTnwJwpweI,535
|
|
104
104
|
atomicshop/etw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
105
|
-
atomicshop/etw/
|
|
106
|
-
atomicshop/etw/
|
|
107
|
-
atomicshop/etw/
|
|
105
|
+
atomicshop/etw/providers.py,sha256=fVmWi-uGdtnsQTDpu_ty6dzx0GMhGokiST73LNBEJ38,129
|
|
106
|
+
atomicshop/etw/sessions.py,sha256=k3miewU278xn829cqDbsuH_bmZHPQE9-Zn-hINbxUSE,1330
|
|
107
|
+
atomicshop/etw/trace.py,sha256=tjBvV4RxCSn8Aoxj8NVxmqHekdECne_y4Zv2lbh11ak,5341
|
|
108
|
+
atomicshop/etw/trace_dns.py,sha256=f4homrWp4qMrmjC9UrEWjr9p9MrUg7SwOVbE22_IYgw,6728
|
|
108
109
|
atomicshop/file_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
109
110
|
atomicshop/file_io/csvs.py,sha256=y8cJtnlN-NNxNupzJgSeGq9aQ4wNxYLFPX9vNNlUiIc,5830
|
|
110
111
|
atomicshop/file_io/docxs.py,sha256=6tcYFGp0vRsHR47VwcRqwhdt2DQOwrAUYhrwN996n9U,5117
|
|
@@ -136,7 +137,7 @@ atomicshop/mitm/engines/__reference_general/responder___reference_general.py,sha
|
|
|
136
137
|
atomicshop/monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
137
138
|
atomicshop/monitor/change_monitor.py,sha256=dGhk5bJPxLCHa2FOVkort99E7vjVojra9GlvhpcKSqE,7551
|
|
138
139
|
atomicshop/monitor/checks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
139
|
-
atomicshop/monitor/checks/dns.py,sha256=
|
|
140
|
+
atomicshop/monitor/checks/dns.py,sha256=YLCFoolq35dnzdnBnfUA2ag-4HfvzcHfRdm3mzX8R8o,7184
|
|
140
141
|
atomicshop/monitor/checks/file.py,sha256=2tIDSlX2KZNc_9i9ji1tcOqupbFTIOj7cKXLyBEDWMk,3263
|
|
141
142
|
atomicshop/monitor/checks/network.py,sha256=CGZWl4WlQrxayZeVF9JspJXwYA-zWx8ECWTVGSlXc98,3825
|
|
142
143
|
atomicshop/monitor/checks/process_running.py,sha256=x66wd6-l466r8sbRQaIli0yswyGt1dH2DVXkGDL6O0Q,1891
|
|
@@ -169,8 +170,8 @@ atomicshop/wrappers/certauthw/certauthw.py,sha256=4WvhjANI7Kzqrr_nKmtA8Kf7B6rute
|
|
|
169
170
|
atomicshop/wrappers/ctyping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
170
171
|
atomicshop/wrappers/ctyping/process_winapi.py,sha256=QcXL-ETtlSSkoT8F7pYle97ubGWsjYp8cx8HxkVMgAc,2762
|
|
171
172
|
atomicshop/wrappers/ctyping/etw_winapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
172
|
-
atomicshop/wrappers/ctyping/etw_winapi/const.py,sha256
|
|
173
|
-
atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py,sha256=
|
|
173
|
+
atomicshop/wrappers/ctyping/etw_winapi/const.py,sha256=jq5nms2qu4FsuXuq3vaHjz9W4ILt9GY-C7CQ8VKcpyg,5764
|
|
174
|
+
atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py,sha256=3DLVXpTeOyTND35T_dKGzKnlLVQ0R3zt3AEcW2bNLNc,5304
|
|
174
175
|
atomicshop/wrappers/ctyping/msi_windows_installer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
175
176
|
atomicshop/wrappers/ctyping/msi_windows_installer/base.py,sha256=Uu9SlWLsQQ6mjE-ek-ptHcmgiI3Ruah9bdZus70EaVY,4884
|
|
176
177
|
atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py,sha256=htzwb2ROYI8yyc82xApStckPS2yCcoyaw32yC15KROs,3285
|
|
@@ -222,7 +223,7 @@ atomicshop/wrappers/loggingw/formatters.py,sha256=mUtcJJfmhLNrwUVYShXTmdu40dBaJu
|
|
|
222
223
|
atomicshop/wrappers/loggingw/handlers.py,sha256=2A_3Qy1B0RvVWZmQocAB6CmpqlXoKJ-yi6iBWG2jNLo,8274
|
|
223
224
|
atomicshop/wrappers/loggingw/loggers.py,sha256=DHOOTAtqkwn1xgvLHSkOiBm6yFGNuQy1kvbhG-TDog8,2374
|
|
224
225
|
atomicshop/wrappers/loggingw/loggingw.py,sha256=m6YySEedP3_4Ik1S_uGMxETSbmRkmMYmAZxhHBlXSlo,16616
|
|
225
|
-
atomicshop/wrappers/loggingw/reading.py,sha256=
|
|
226
|
+
atomicshop/wrappers/loggingw/reading.py,sha256=bsSUM9_epMO2L-lHBEULFxeqdxXOHfICt-1BtQZn7lA,16712
|
|
226
227
|
atomicshop/wrappers/nodejsw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
227
228
|
atomicshop/wrappers/nodejsw/install_nodejs.py,sha256=QZg-R2iTQt7kFb8wNtnTmwraSGwvUs34JIasdbNa7ZU,5154
|
|
228
229
|
atomicshop/wrappers/playwrightw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -259,8 +260,8 @@ atomicshop/wrappers/socketw/socket_server_tester.py,sha256=AhpurHJmP2kgzHaUbq5ey
|
|
|
259
260
|
atomicshop/wrappers/socketw/socket_wrapper.py,sha256=aXBwlEIJhFT0-c4i8iNlFx2It9VpCEpsv--5Oqcpxao,11624
|
|
260
261
|
atomicshop/wrappers/socketw/ssl_base.py,sha256=k4V3gwkbq10MvOH4btU4onLX2GNOsSfUAdcHmL1rpVE,2274
|
|
261
262
|
atomicshop/wrappers/socketw/statistics_csv.py,sha256=t3dtDEfN47CfYVi0CW6Kc2QHTEeZVyYhc57IYYh5nmA,826
|
|
262
|
-
atomicshop-2.12.
|
|
263
|
-
atomicshop-2.12.
|
|
264
|
-
atomicshop-2.12.
|
|
265
|
-
atomicshop-2.12.
|
|
266
|
-
atomicshop-2.12.
|
|
263
|
+
atomicshop-2.12.26.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
|
|
264
|
+
atomicshop-2.12.26.dist-info/METADATA,sha256=HYm5lvYyf77XdD_MWANy8aWX2kpFD7kD-EmOZg19yOQ,10479
|
|
265
|
+
atomicshop-2.12.26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
266
|
+
atomicshop-2.12.26.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
|
|
267
|
+
atomicshop-2.12.26.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|