atomicshop 2.19.10__py3-none-any.whl → 2.19.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/a_installs/win/robocorp.py +3 -0
- atomicshop/console_user_response.py +7 -14
- atomicshop/etws/trace.py +10 -22
- atomicshop/etws/traces/trace_dns.py +1 -1
- atomicshop/etws/traces/trace_tcp.py +130 -0
- atomicshop/process_poller/simple_process_pool.py +41 -5
- atomicshop/versioning.py +27 -0
- atomicshop/web.py +2 -0
- atomicshop/wrappers/githubw.py +88 -36
- {atomicshop-2.19.10.dist-info → atomicshop-2.19.12.dist-info}/METADATA +1 -1
- {atomicshop-2.19.10.dist-info → atomicshop-2.19.12.dist-info}/RECORD +15 -13
- {atomicshop-2.19.10.dist-info → atomicshop-2.19.12.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.19.10.dist-info → atomicshop-2.19.12.dist-info}/WHEEL +0 -0
- {atomicshop-2.19.10.dist-info → atomicshop-2.19.12.dist-info}/top_level.txt +0 -0
atomicshop/__init__.py
CHANGED
|
@@ -41,6 +41,9 @@ def main():
|
|
|
41
41
|
print_api("Initializing Robocorp Browser.")
|
|
42
42
|
subprocess.check_call(["rfbrowser", "init"])
|
|
43
43
|
|
|
44
|
+
print_api("Installing Additional modules.")
|
|
45
|
+
subprocess.check_call(["pip", "install", "--upgrade", "matplotlib", "imagehash"])
|
|
46
|
+
|
|
44
47
|
print_api("Installing Tesseract OCR.")
|
|
45
48
|
github_wrapper = githubw.GitHubWrapper(
|
|
46
49
|
user_name="tesseract-ocr",
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
# v1.0.2 - 26.03.2023 20:40
|
|
2
1
|
import sys
|
|
3
2
|
|
|
4
3
|
|
|
5
|
-
def query_positive_negative(
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
def query_positive_negative(
|
|
5
|
+
question_string: str,
|
|
6
|
+
add_first_values_to_question: bool = True,
|
|
7
|
+
positive_answers: list = None,
|
|
8
|
+
negative_answers: list = None
|
|
9
|
+
) -> bool:
|
|
9
10
|
"""
|
|
10
11
|
Ask for "yes" / "no" input to a question that is passed as a "question_string".
|
|
11
12
|
Returns 'True' for 'positive' answers and 'False' for 'negative' answers.
|
|
@@ -31,10 +32,8 @@ def query_positive_negative(question_string: str,
|
|
|
31
32
|
if add_first_values_to_question:
|
|
32
33
|
question_string = f'{question_string} [{positive_answers[0]}/{negative_answers[0]}]'
|
|
33
34
|
|
|
34
|
-
# Defining variable as False for While loop to run
|
|
35
|
-
right_answer = False
|
|
36
35
|
# As long as "right_answer" is False the loop will execute again
|
|
37
|
-
while
|
|
36
|
+
while True:
|
|
38
37
|
# Print the passed question
|
|
39
38
|
print(question_string)
|
|
40
39
|
# Get the input from the console in lowercase
|
|
@@ -42,22 +41,16 @@ def query_positive_negative(question_string: str,
|
|
|
42
41
|
|
|
43
42
|
# If the gathered value is in "Yes" answers array
|
|
44
43
|
if choice in positive_answers:
|
|
45
|
-
# "right_answer" variable is True, so the loop will not execute again
|
|
46
|
-
right_answer = True
|
|
47
44
|
# Function will return True
|
|
48
45
|
return True
|
|
49
46
|
# Else If the gathered value is in "No" answers array
|
|
50
47
|
elif choice in negative_answers:
|
|
51
|
-
# "right_answer" variable is True, so the loop will not execute again
|
|
52
|
-
right_answer = True
|
|
53
48
|
# Function will return False
|
|
54
49
|
return False
|
|
55
50
|
# If the gathered input is not in the arrays
|
|
56
51
|
else:
|
|
57
52
|
# Then output to console the message
|
|
58
53
|
print("Please respond with either:", positive_answers, negative_answers)
|
|
59
|
-
# "right_answer" variable stays False, so the loop will execute again
|
|
60
|
-
right_answer = False
|
|
61
54
|
|
|
62
55
|
|
|
63
56
|
def do_you_want_to_continue_yn(message: str) -> None:
|
atomicshop/etws/trace.py
CHANGED
|
@@ -83,7 +83,7 @@ class EventTrace(etw.ETW):
|
|
|
83
83
|
|
|
84
84
|
self.self_hosted_poller: bool = False
|
|
85
85
|
if self.enable_process_poller:
|
|
86
|
-
if
|
|
86
|
+
if self.process_pool_shared_dict_proxy is None:
|
|
87
87
|
self.self_hosted_poller = True
|
|
88
88
|
self.process_poller = simple_process_pool.SimpleProcessPool()
|
|
89
89
|
self.multiprocessing_manager: multiprocessing.managers.SyncManager = multiprocessing.Manager()
|
|
@@ -98,7 +98,7 @@ class EventTrace(etw.ETW):
|
|
|
98
98
|
)
|
|
99
99
|
|
|
100
100
|
def start(self):
|
|
101
|
-
if self.enable_process_poller:
|
|
101
|
+
if self.enable_process_poller and self.self_hosted_poller:
|
|
102
102
|
self.process_poller.start()
|
|
103
103
|
|
|
104
104
|
# Check if the session name already exists.
|
|
@@ -123,11 +123,10 @@ class EventTrace(etw.ETW):
|
|
|
123
123
|
def stop(self):
|
|
124
124
|
super().stop()
|
|
125
125
|
|
|
126
|
-
if self.
|
|
126
|
+
if self.self_hosted_poller:
|
|
127
127
|
self.process_poller.stop()
|
|
128
128
|
|
|
129
|
-
|
|
130
|
-
self.multiprocessing_manager.shutdown()
|
|
129
|
+
self.multiprocessing_manager.shutdown()
|
|
131
130
|
|
|
132
131
|
def emit(self):
|
|
133
132
|
"""
|
|
@@ -146,32 +145,21 @@ class EventTrace(etw.ETW):
|
|
|
146
145
|
:return: etw event object.
|
|
147
146
|
"""
|
|
148
147
|
|
|
149
|
-
# Get the processes first, since we need the process name and command line.
|
|
150
|
-
# If they're not ready, we will get just pids from DNS tracing.
|
|
151
|
-
if self.enable_process_poller:
|
|
152
|
-
self._get_processes_from_poller()
|
|
153
|
-
|
|
154
148
|
event: tuple = self.event_queue.get()
|
|
155
149
|
|
|
156
150
|
event_dict: dict = {
|
|
157
151
|
'EventId': event[0],
|
|
158
|
-
'EventHeader': event[1]
|
|
159
|
-
'pid': event[1]['EventHeader']['ProcessId']
|
|
152
|
+
'EventHeader': event[1]
|
|
160
153
|
}
|
|
161
154
|
|
|
155
|
+
if 'ProcessId' not in event[1]:
|
|
156
|
+
event_dict['pid'] = event[1]['EventHeader']['ProcessId']
|
|
157
|
+
else:
|
|
158
|
+
event_dict['pid'] = event[1]['ProcessId']
|
|
159
|
+
|
|
162
160
|
if self.enable_process_poller:
|
|
163
161
|
process_info: dict = self.pid_process_converter.get_process_by_pid(event_dict['pid'])
|
|
164
162
|
event_dict['name'] = process_info['name']
|
|
165
163
|
event_dict['cmdline'] = process_info['cmdline']
|
|
166
164
|
|
|
167
165
|
return event_dict
|
|
168
|
-
|
|
169
|
-
def _get_processes_from_poller(self):
|
|
170
|
-
processes: dict = {}
|
|
171
|
-
while not processes:
|
|
172
|
-
processes = self.process_poller.get_processes()
|
|
173
|
-
|
|
174
|
-
if isinstance(processes, BaseException):
|
|
175
|
-
raise processes
|
|
176
|
-
|
|
177
|
-
return processes
|
|
@@ -47,7 +47,7 @@ class DnsRequestResponseTrace:
|
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
dns_trace_w = dns_trace.DnsTrace(
|
|
50
|
-
attrs=['pid', 'name', 'cmdline', '
|
|
50
|
+
attrs=['pid', 'name', 'cmdline', 'query', 'query_type'],
|
|
51
51
|
session_name='MyDnsTrace',
|
|
52
52
|
close_existing_session_name=True,
|
|
53
53
|
enable_process_poller=True
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import multiprocessing.managers
|
|
2
|
+
|
|
3
|
+
from .. import trace, const, providers
|
|
4
|
+
from ...basics import dicts
|
|
5
|
+
from ... import dns, ip_addresses
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
ETW_DEFAULT_SESSION_NAME: str = 'AtomicShopTcpTrace'
|
|
9
|
+
|
|
10
|
+
PROVIDER_NAME: str = "Microsoft-Windows-TCPIP"
|
|
11
|
+
PROVIDER_GUID: str = '{' + providers.get_provider_guid_by_name(PROVIDER_NAME) + '}'
|
|
12
|
+
REQUEST_RESP_EVENT_ID: int = 1033
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TcpIpNewConnectionsTrace:
|
|
16
|
+
"""
|
|
17
|
+
TcpIpNewConnectionsTrace class use to trace new connection events from Windows Event Tracing:
|
|
18
|
+
Provider: Microsoft-Windows-TCPIP
|
|
19
|
+
EventId: 1033
|
|
20
|
+
"""
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
attrs: list = None,
|
|
24
|
+
session_name: str = None,
|
|
25
|
+
close_existing_session_name: bool = True,
|
|
26
|
+
process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy = None
|
|
27
|
+
):
|
|
28
|
+
"""
|
|
29
|
+
:param attrs: List of attributes to return. If None, all attributes will be returned.
|
|
30
|
+
:param session_name: The name of the session to create. If not provided, a UUID will be generated.
|
|
31
|
+
:param close_existing_session_name: Boolean to close existing session names.
|
|
32
|
+
True: if ETW session with 'session_name' exists, you will be notified and the session will be closed.
|
|
33
|
+
Then the new session with this name will be created.
|
|
34
|
+
False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
|
|
35
|
+
created. Instead, the existing session will be used. If there is a buffer from the previous session,
|
|
36
|
+
you will get the events from the buffer.
|
|
37
|
+
:param process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy, multiprocessing shared dict proxy
|
|
38
|
+
that contains current processes.
|
|
39
|
+
Check the 'atomicshop\process_poller\simple_process_pool.py' SimpleProcessPool class for more information.
|
|
40
|
+
|
|
41
|
+
For this specific class it means that you can run the process poller outside of this class and pass the
|
|
42
|
+
'process_pool_shared_dict_proxy' to this class. Then you can get the process name and command line for
|
|
43
|
+
the DNS events from the 'process_pool_shared_dict_proxy' and use it also in other classes.
|
|
44
|
+
|
|
45
|
+
-------------------------------------------------
|
|
46
|
+
|
|
47
|
+
Usage Example:
|
|
48
|
+
from atomicshop.etw import tcp_trace
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
tcp_trace_w = tcp_trace.TcpIpNewConnectionsTrace(
|
|
52
|
+
attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
|
|
53
|
+
session_name='MyTcpTrace',
|
|
54
|
+
close_existing_session_name=True,
|
|
55
|
+
enable_process_poller=True
|
|
56
|
+
)
|
|
57
|
+
tcp_trace_w.start()
|
|
58
|
+
while True:
|
|
59
|
+
tcp_dict = tcp_trace_w.emit()
|
|
60
|
+
print(tcp_dict)
|
|
61
|
+
tcp_trace_w.stop()
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
self.attrs = attrs
|
|
65
|
+
self.process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy = process_pool_shared_dict_proxy
|
|
66
|
+
|
|
67
|
+
if not session_name:
|
|
68
|
+
session_name = ETW_DEFAULT_SESSION_NAME
|
|
69
|
+
|
|
70
|
+
self.event_trace = trace.EventTrace(
|
|
71
|
+
providers=[(PROVIDER_NAME, PROVIDER_GUID)],
|
|
72
|
+
# lambda x: self.event_queue.put(x),
|
|
73
|
+
event_id_filters=[REQUEST_RESP_EVENT_ID],
|
|
74
|
+
session_name=session_name,
|
|
75
|
+
close_existing_session_name=close_existing_session_name,
|
|
76
|
+
enable_process_poller=True,
|
|
77
|
+
process_pool_shared_dict_proxy=self.process_pool_shared_dict_proxy
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def start(self):
|
|
81
|
+
self.event_trace.start()
|
|
82
|
+
|
|
83
|
+
def stop(self):
|
|
84
|
+
self.event_trace.stop()
|
|
85
|
+
|
|
86
|
+
def emit(self):
|
|
87
|
+
"""
|
|
88
|
+
Function that will return the next event from the queue.
|
|
89
|
+
The queue is blocking, so if there is no event in the queue, the function will wait until there is one.
|
|
90
|
+
|
|
91
|
+
Usage Example:
|
|
92
|
+
while True:
|
|
93
|
+
tcp_dict = tcp_trace.emit()
|
|
94
|
+
print(tcp_dict)
|
|
95
|
+
|
|
96
|
+
:return: Dictionary with the event data.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
# Get the event from ETW as is.
|
|
100
|
+
event = self.event_trace.emit()
|
|
101
|
+
|
|
102
|
+
local_address_port: str = event['EventHeader']['LocalAddress']
|
|
103
|
+
remote_address_port: str = event['EventHeader']['RemoteAddress']
|
|
104
|
+
|
|
105
|
+
if 'ffff' in local_address_port:
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
local_address, local_port = local_address_port.rsplit(':', 1)
|
|
109
|
+
local_address = local_address.replace('[', '').replace(']', '')
|
|
110
|
+
|
|
111
|
+
remote_address, remote_port = remote_address_port.rsplit(':', 1)
|
|
112
|
+
remote_address = remote_address.replace('[', '').replace(']', '')
|
|
113
|
+
|
|
114
|
+
event_dict: dict = {
|
|
115
|
+
'event_id': event['EventId'],
|
|
116
|
+
'local_ip': local_address,
|
|
117
|
+
'local_port': local_port,
|
|
118
|
+
'remote_ip': remote_address,
|
|
119
|
+
'remote_port': remote_port,
|
|
120
|
+
'status': event['EventHeader']['Status'],
|
|
121
|
+
'pid': event['pid'],
|
|
122
|
+
'name': event['name'],
|
|
123
|
+
'cmdline': event['cmdline']
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if self.attrs:
|
|
127
|
+
event_dict = dicts.reorder_keys(
|
|
128
|
+
event_dict, self.attrs, skip_keys_not_in_list=True)
|
|
129
|
+
|
|
130
|
+
return event_dict
|
|
@@ -23,6 +23,40 @@ class SimpleProcessPool:
|
|
|
23
23
|
The idea is similar to the process_poller.process_pool.ProcessPool class, but this class is simpler and uses
|
|
24
24
|
only the pywin32 tracing of the Windows Event Log Process Creation and Process Termination events.
|
|
25
25
|
The simple process pool is used to get things simpler than the process_pool.ProcessPool class.
|
|
26
|
+
|
|
27
|
+
Example of starting the process pool in multiprocess:
|
|
28
|
+
import sys
|
|
29
|
+
|
|
30
|
+
from atomicshop.process_poller import simple_process_pool
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def start_process_pool(process_pool_shared_dict_proxy):
|
|
34
|
+
process_poller = simple_process_pool.SimpleProcessPool(
|
|
35
|
+
process_pool_shared_dict_proxy=process_pool_shared_dict_proxy)
|
|
36
|
+
process_poller.start()
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
# Keep the process alive.
|
|
40
|
+
while True:
|
|
41
|
+
time.sleep(1)
|
|
42
|
+
except KeyboardInterrupt:
|
|
43
|
+
process_poller.stop()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def main():
|
|
47
|
+
# Create the shared multiprocessing dictionary of the process pool.
|
|
48
|
+
manager = multiprocessing.Manager()
|
|
49
|
+
multiprocess_dict_proxy = manager.dict()
|
|
50
|
+
|
|
51
|
+
# Start the process pool in a separate process.
|
|
52
|
+
pool_process = multiprocessing.Process(target=start_process_pool, args=(multiprocess_dict_proxy,))
|
|
53
|
+
pool_process.start()
|
|
54
|
+
|
|
55
|
+
# Pass the shared dict proxy to other functions.
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == '__main__':
|
|
59
|
+
sys.exit(main())
|
|
26
60
|
"""
|
|
27
61
|
|
|
28
62
|
def __init__(
|
|
@@ -79,7 +113,7 @@ class SimpleProcessPool:
|
|
|
79
113
|
self._processes = get_process_list.GetProcessList(
|
|
80
114
|
get_method='pywin32', connect_on_init=True).get_processes(as_dict=True)
|
|
81
115
|
|
|
82
|
-
thread_get_queue = threading.Thread(target=self._start_main_thread)
|
|
116
|
+
thread_get_queue = threading.Thread(target=self._start_main_thread, args=(self.process_pool_shared_dict_proxy,))
|
|
83
117
|
thread_get_queue.daemon = True
|
|
84
118
|
thread_get_queue.start()
|
|
85
119
|
|
|
@@ -93,7 +127,7 @@ class SimpleProcessPool:
|
|
|
93
127
|
def get_processes(self):
|
|
94
128
|
return self._processes
|
|
95
129
|
|
|
96
|
-
def _start_main_thread(self):
|
|
130
|
+
def _start_main_thread(self, process_pool_shared_dict_proxy):
|
|
97
131
|
get_instance = process_create.ProcessCreateSubscriber()
|
|
98
132
|
get_instance.start()
|
|
99
133
|
|
|
@@ -109,9 +143,9 @@ class SimpleProcessPool:
|
|
|
109
143
|
}
|
|
110
144
|
|
|
111
145
|
# Update the multiprocessing shared dict proxy.
|
|
112
|
-
if
|
|
113
|
-
|
|
114
|
-
|
|
146
|
+
if process_pool_shared_dict_proxy is not None:
|
|
147
|
+
process_pool_shared_dict_proxy.clear()
|
|
148
|
+
process_pool_shared_dict_proxy.update(self._processes)
|
|
115
149
|
|
|
116
150
|
# print_api(f'Process [{process_id}] added to the pool.', color='blue')
|
|
117
151
|
|
|
@@ -186,6 +220,7 @@ class PidProcessConverter:
|
|
|
186
220
|
process_dict: dict = dict()
|
|
187
221
|
while counter < WAIT_FOR_PROCESS_POLLER_PID_COUNTS:
|
|
188
222
|
if pid not in self.process_pool_shared_dict_proxy:
|
|
223
|
+
# print(dict(self.process_pool_shared_dict_proxy))
|
|
189
224
|
time.sleep(0.1)
|
|
190
225
|
counter += 1
|
|
191
226
|
else:
|
|
@@ -193,6 +228,7 @@ class PidProcessConverter:
|
|
|
193
228
|
break
|
|
194
229
|
|
|
195
230
|
if counter == WAIT_FOR_PROCESS_POLLER_PID_COUNTS and not process_dict:
|
|
231
|
+
print_api(f"Error: The PID [{pid}] is not in the pool, trying DLL snapshot.", color='yellow')
|
|
196
232
|
# Last resort, try to get the process name by current process snapshot.
|
|
197
233
|
processes = self.get_process_with_dll_instance.get_process_details(as_dict=True)
|
|
198
234
|
if pid not in processes:
|
atomicshop/versioning.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def is_target_newer(
|
|
5
|
+
base_version: Union[str, tuple],
|
|
6
|
+
target_version: Union[str, tuple]
|
|
7
|
+
):
|
|
8
|
+
"""
|
|
9
|
+
Check if the target version is newer than the base version.
|
|
10
|
+
Example: is_target_newer('1.0.0', '1.0.1') -> True
|
|
11
|
+
Example: is_target_newer('1.0.0', '1.0.0') -> False
|
|
12
|
+
Example: is_target_newer('1.0.1', '1.0.0') -> False
|
|
13
|
+
Example: is_target_newer('1.0.1', '1.0.2') -> True
|
|
14
|
+
Example: is_target_newer((1,0,1), (1,1,0)) -> True
|
|
15
|
+
|
|
16
|
+
:param base_version: The base version to compare against.
|
|
17
|
+
:param target_version: The target version to compare.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# Convert string to tuple if string was passed.
|
|
21
|
+
if isinstance(base_version, str):
|
|
22
|
+
base_version = tuple(map(int, base_version.split('.')))
|
|
23
|
+
if isinstance(target_version, str):
|
|
24
|
+
target_version = tuple(map(int, target_version.split('.')))
|
|
25
|
+
|
|
26
|
+
# Compare the versions.
|
|
27
|
+
return target_version > base_version
|
atomicshop/web.py
CHANGED
|
@@ -233,6 +233,8 @@ def download(
|
|
|
233
233
|
|
|
234
234
|
if aggregated_bytes_int == file_size_bytes_int:
|
|
235
235
|
print_api.print_api(f'Successfully Downloaded to: {file_path}', color="green", **kwargs)
|
|
236
|
+
elif file_size_bytes_int is None:
|
|
237
|
+
pass
|
|
236
238
|
else:
|
|
237
239
|
message = f'Download failed: {aggregated_bytes_int} / {file_size_bytes_int}. File: {file_path}'
|
|
238
240
|
print_api.print_api(
|
atomicshop/wrappers/githubw.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import requests
|
|
2
2
|
import fnmatch
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
from typing import Literal
|
|
3
6
|
|
|
4
|
-
from .. import web, urls
|
|
7
|
+
from .. import web, urls, filesystem
|
|
5
8
|
from ..print_api import print_api
|
|
6
9
|
|
|
7
10
|
|
|
@@ -21,7 +24,9 @@ class GitHubWrapper:
|
|
|
21
24
|
branch: str = 'master',
|
|
22
25
|
path: str = None,
|
|
23
26
|
pat: str = None,
|
|
24
|
-
branch_file_extension:
|
|
27
|
+
branch_file_extension: Literal[
|
|
28
|
+
'zip',
|
|
29
|
+
'tar.gz'] = 'zip'
|
|
25
30
|
):
|
|
26
31
|
"""
|
|
27
32
|
This class is a wrapper for GitHub repositories. It can download the branch file from the repository and extract
|
|
@@ -37,8 +42,8 @@ class GitHubWrapper:
|
|
|
37
42
|
:param path: str, the path to the file/folder inside the repo that we'll do certain actions on.
|
|
38
43
|
Actions example: get_latest_commit_comment, download_path_from_branch.
|
|
39
44
|
:param pat: str, the personal access token to the repo.
|
|
40
|
-
:param branch_file_extension: str, the branch file extension. The default is '
|
|
41
|
-
You also can use '
|
|
45
|
+
:param branch_file_extension: str, the branch file extension. The default is 'zip'.
|
|
46
|
+
You also can use 'tar.gz' as extension.
|
|
42
47
|
|
|
43
48
|
================================================================================================================
|
|
44
49
|
Usage to download the 'master' branch file:
|
|
@@ -93,13 +98,19 @@ class GitHubWrapper:
|
|
|
93
98
|
self.branch_file_extension: str = branch_file_extension
|
|
94
99
|
|
|
95
100
|
# Default variables.
|
|
96
|
-
self.
|
|
97
|
-
|
|
101
|
+
if self.branch_file_extension == 'zip':
|
|
102
|
+
self.branch_type_directory: str = 'zipball'
|
|
103
|
+
elif self.branch_file_extension == 'tar.gz':
|
|
104
|
+
self.branch_type_directory: str = 'tarball'
|
|
105
|
+
else:
|
|
106
|
+
raise ValueError(f"Unsupported branch file extension: {self.branch_file_extension}")
|
|
107
|
+
|
|
98
108
|
self.domain: str = 'github.com'
|
|
99
109
|
|
|
100
110
|
# Initialize variables.
|
|
101
111
|
self.branch_download_link: str = str()
|
|
102
112
|
self.branch_downloaded_folder_name: str = str()
|
|
113
|
+
self.branch_file_name: str = str()
|
|
103
114
|
self.api_url: str = str()
|
|
104
115
|
self.latest_release_json_url: str = str()
|
|
105
116
|
self.commits_url: str = str()
|
|
@@ -126,14 +137,15 @@ class GitHubWrapper:
|
|
|
126
137
|
raise ValueError("'user_name' or 'repo_name' is empty.")
|
|
127
138
|
|
|
128
139
|
self.repo_url = f'https://{self.domain}/{self.user_name}/{self.repo_name}'
|
|
129
|
-
self.branch_download_link = f'{self.repo_url}/{self.archive_directory}/{self.branch_file_name}'
|
|
130
140
|
self.branch_downloaded_folder_name = f'{self.repo_name}-{self.branch}'
|
|
141
|
+
self.branch_file_name: str = f'{self.repo_name}-{self.branch}.{self.branch_file_extension}'
|
|
131
142
|
|
|
132
143
|
self.api_url = f'https://api.{self.domain}/repos/{self.user_name}/{self.repo_name}'
|
|
133
144
|
|
|
134
145
|
self.latest_release_json_url: str = f'{self.api_url}/releases/latest'
|
|
135
146
|
self.commits_url: str = f'{self.api_url}/commits'
|
|
136
147
|
self.contents_url: str = f'{self.api_url}/contents'
|
|
148
|
+
self.branch_download_link = f'{self.api_url}/{self.branch_type_directory}/{self.branch}'
|
|
137
149
|
|
|
138
150
|
def build_links_from_repo_url(self, **kwargs):
|
|
139
151
|
if not self.repo_url:
|
|
@@ -178,48 +190,88 @@ class GitHubWrapper:
|
|
|
178
190
|
:return:
|
|
179
191
|
"""
|
|
180
192
|
|
|
181
|
-
|
|
193
|
+
def download_file(file_url: str, target_dir: str, file_name: str, current_headers: dict) -> None:
|
|
194
|
+
os.makedirs(target_dir, exist_ok=True)
|
|
182
195
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
contents_url = f"{self.contents_url}/{self.path}"
|
|
196
|
+
web.download(
|
|
197
|
+
file_url=file_url,
|
|
198
|
+
target_directory=target_dir,
|
|
199
|
+
file_name=file_name,
|
|
200
|
+
headers=current_headers
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
def download_directory(folder_path: str, target_dir: str, current_headers: dict) -> None:
|
|
204
|
+
# Construct the API URL for the current folder.
|
|
205
|
+
contents_url = f"{self.contents_url}/{folder_path}"
|
|
194
206
|
params = {'ref': self.branch}
|
|
195
207
|
|
|
196
|
-
response = requests.get(contents_url, headers=
|
|
208
|
+
response = requests.get(contents_url, headers=current_headers, params=params)
|
|
197
209
|
response.raise_for_status()
|
|
198
210
|
|
|
211
|
+
# Get the list of items (files and subdirectories) in the folder.
|
|
199
212
|
items = response.json()
|
|
200
213
|
|
|
201
|
-
# Ensure the target directory exists.
|
|
202
|
-
os.makedirs(
|
|
214
|
+
# Ensure the local target directory exists.
|
|
215
|
+
os.makedirs(target_dir, exist_ok=True)
|
|
203
216
|
|
|
217
|
+
# Process each item.
|
|
204
218
|
for item in items:
|
|
205
|
-
|
|
219
|
+
local_item_path = os.path.join(target_dir, item['name'])
|
|
206
220
|
if item['type'] == 'file':
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
download(
|
|
211
|
-
file_url=file_url,
|
|
212
|
-
target_directory=target_directory,
|
|
221
|
+
download_file(
|
|
222
|
+
file_url=item['download_url'],
|
|
223
|
+
target_dir=target_dir,
|
|
213
224
|
file_name=item['name'],
|
|
214
|
-
|
|
225
|
+
current_headers=current_headers
|
|
215
226
|
)
|
|
216
227
|
elif item['type'] == 'dir':
|
|
217
228
|
# Recursively download subdirectories.
|
|
218
|
-
|
|
219
|
-
folder_path=
|
|
220
|
-
|
|
229
|
+
download_directory(
|
|
230
|
+
folder_path=f"{folder_path}/{item['name']}",
|
|
231
|
+
target_dir=local_item_path,
|
|
232
|
+
current_headers=current_headers
|
|
221
233
|
)
|
|
222
234
|
|
|
235
|
+
headers: dict = self._get_headers()
|
|
236
|
+
|
|
237
|
+
if not download_each_file:
|
|
238
|
+
if self.path:
|
|
239
|
+
download_target_directory = tempfile.mkdtemp()
|
|
240
|
+
current_archive_remove_first_directory = True
|
|
241
|
+
else:
|
|
242
|
+
download_target_directory = target_directory
|
|
243
|
+
current_archive_remove_first_directory = archive_remove_first_directory
|
|
244
|
+
|
|
245
|
+
# Download the repo to current working directory, extract and remove the archive.
|
|
246
|
+
web.download_and_extract_file(
|
|
247
|
+
file_url=self.branch_download_link,
|
|
248
|
+
file_name=self.branch_file_name,
|
|
249
|
+
target_directory=download_target_directory,
|
|
250
|
+
archive_remove_first_directory=current_archive_remove_first_directory,
|
|
251
|
+
headers=headers,
|
|
252
|
+
**kwargs)
|
|
253
|
+
|
|
254
|
+
if self.path:
|
|
255
|
+
source_path: str = f"{download_target_directory}{os.sep}{self.path}"
|
|
256
|
+
|
|
257
|
+
if not archive_remove_first_directory:
|
|
258
|
+
target_directory = os.path.join(target_directory, self.path)
|
|
259
|
+
filesystem.create_directory(target_directory)
|
|
260
|
+
|
|
261
|
+
# Move the path to the target directory.
|
|
262
|
+
filesystem.move_folder_contents_to_folder(
|
|
263
|
+
source_path, target_directory)
|
|
264
|
+
|
|
265
|
+
# Remove the downloaded branch directory.
|
|
266
|
+
filesystem.remove_directory(download_target_directory)
|
|
267
|
+
else:
|
|
268
|
+
if archive_remove_first_directory:
|
|
269
|
+
current_target_directory = target_directory
|
|
270
|
+
else:
|
|
271
|
+
current_target_directory = os.path.join(target_directory, self.path)
|
|
272
|
+
|
|
273
|
+
download_directory(self.path, current_target_directory, headers)
|
|
274
|
+
|
|
223
275
|
def get_latest_release_url(
|
|
224
276
|
self,
|
|
225
277
|
string_pattern: str,
|
|
@@ -430,8 +482,7 @@ def github_wrapper_main(
|
|
|
430
482
|
|
|
431
483
|
if download_branch:
|
|
432
484
|
git_wrapper.download_and_extract_branch(
|
|
433
|
-
target_directory=target_directory, download_each_file=
|
|
434
|
-
archive_remove_first_directory=True)
|
|
485
|
+
target_directory=target_directory, download_each_file=False, archive_remove_first_directory=True)
|
|
435
486
|
|
|
436
487
|
return 0
|
|
437
488
|
|
|
@@ -445,5 +496,6 @@ def github_wrapper_main_with_args():
|
|
|
445
496
|
path=args.path,
|
|
446
497
|
target_directory=args.target_directory,
|
|
447
498
|
pat=args.pat,
|
|
448
|
-
get_latest_commit_comment=args.get_latest_commit_comment
|
|
499
|
+
get_latest_commit_comment=args.get_latest_commit_comment,
|
|
500
|
+
download_branch=args.download_branch
|
|
449
501
|
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
atomicshop/__init__.py,sha256=
|
|
1
|
+
atomicshop/__init__.py,sha256=XtGdtCQBXBDdG8_6wWA54tuzPJ5IE_hCw_e_N2FoXLM,124
|
|
2
2
|
atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
|
|
3
3
|
atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
|
|
4
4
|
atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
|
|
@@ -7,7 +7,7 @@ atomicshop/certificates.py,sha256=MEGj7t3Nt2CHE7yzXrvFTLCOKZG9tJ6Ok5JC2BsFRis,76
|
|
|
7
7
|
atomicshop/command_line_processing.py,sha256=u5yT9Ger_cu7ni5ID0VFlRbVD46ARHeNC9tRM-_YXrQ,1038
|
|
8
8
|
atomicshop/config_init.py,sha256=50kD2lXP8sgwPekcmAbfADcY46YvXkF-6XIdA7W_638,2501
|
|
9
9
|
atomicshop/console_output.py,sha256=AOSJjrRryE97PAGtgDL03IBtWSi02aNol8noDnW3k6M,4667
|
|
10
|
-
atomicshop/console_user_response.py,sha256=
|
|
10
|
+
atomicshop/console_user_response.py,sha256=OHcjuzWAys6WmfRnMIU_nkJA634kKmJh6T8w1VtUTJM,2714
|
|
11
11
|
atomicshop/datetimes.py,sha256=IQZ66lmta-ZqxYbyHzm_9eugbJFSilXK1e0kfMgoXGg,18371
|
|
12
12
|
atomicshop/diff_check.py,sha256=vxTDccVbGZHEge6Ja9_ArLWwslOUgIoJAdYPylh4cZg,27176
|
|
13
13
|
atomicshop/dns.py,sha256=5Gimq_WY2arqg7BeGmR7P--fGfnH0Dsh8lrOt_H0jRY,6817
|
|
@@ -44,8 +44,9 @@ atomicshop/timer.py,sha256=7Zw1KRV0acHCRATMnanyX2MLBb63Hc-6us3rCZ9dNlY,2345
|
|
|
44
44
|
atomicshop/urls.py,sha256=aJ0NGS9qqaKeqjkkWBs80jaBBg6MYBiPuLIyPGxscVc,1557
|
|
45
45
|
atomicshop/uuids.py,sha256=JSQdm3ZTJiwPQ1gYe6kU0TKS_7suwVrHc8JZDGYlydM,2214
|
|
46
46
|
atomicshop/venvs.py,sha256=D9lwOoObkYoRx-weuoAmbvN-RdSHhVm4DE9TVl-utAs,903
|
|
47
|
+
atomicshop/versioning.py,sha256=e5W6m9AF3__M5nntqI9CqNAeHqkwY9JhlnpYeZ1CEus,970
|
|
47
48
|
atomicshop/virtualization.py,sha256=LPP4vjE0Vr10R6DA4lqhfX_WaNdDGRAZUW0Am6VeGco,494
|
|
48
|
-
atomicshop/web.py,sha256=
|
|
49
|
+
atomicshop/web.py,sha256=uIYt1WHXllc2wwnsW4AOI_IrAEm9j89qAQa4v3nR8Wk,12105
|
|
49
50
|
atomicshop/websocket_parse.py,sha256=aLHWyKqaYqEn_MRBWm2L6rIl6QPmqbVrjEXE_rBzwCw,16711
|
|
50
51
|
atomicshop/a_installs/ubuntu/docker_rootless.py,sha256=9IPNtGZYjfy1_n6ZRt7gWz9KZgR6XCgevjqq02xk-o0,281
|
|
51
52
|
atomicshop/a_installs/ubuntu/docker_sudo.py,sha256=JzayxeyKDtiuT4Icp2L2LyFRbx4wvpyN_bHLfZ-yX5E,281
|
|
@@ -57,7 +58,7 @@ atomicshop/a_installs/win/fibratus.py,sha256=TU4e9gdZ_zI73C40uueJ59pD3qmN-UFGdX5
|
|
|
57
58
|
atomicshop/a_installs/win/mongodb.py,sha256=AqyItXu19aaoe49pppDxtEkXey6PMy0PoT2Y_RmPpPE,179
|
|
58
59
|
atomicshop/a_installs/win/nodejs.py,sha256=U519Dyt4bsQPbEg_PwnZL5tsbfqDr1BbhxwoQFZsSKo,200
|
|
59
60
|
atomicshop/a_installs/win/pycharm.py,sha256=j_RSd7aDOyC3yDd-_GUTMLlQTmDrqtVFG--oUfGLiZk,140
|
|
60
|
-
atomicshop/a_installs/win/robocorp.py,sha256=
|
|
61
|
+
atomicshop/a_installs/win/robocorp.py,sha256=ExdMR705fW4iyNDZnH9YZMrio4FYEwNH_8nEP1GWkjM,2383
|
|
61
62
|
atomicshop/a_installs/win/wsl_ubuntu_lts.py,sha256=dZbPRLNKFeMd6MotjkE6UDY9cOiIaaclIdR1kGYWI50,139
|
|
62
63
|
atomicshop/a_mains/dns_gateway_setting.py,sha256=ncc2rFQCChxlNP59UshwmTonLqC6MWblrVAzbbz-13M,149
|
|
63
64
|
atomicshop/a_mains/github_wrapper.py,sha256=F-PoZknVCxWPN0PTO6l7ZNiaYvo7OVFKFI_zlPt56ps,169
|
|
@@ -117,10 +118,11 @@ atomicshop/etws/_pywintrace_fix.py,sha256=nHrtnAb796eOZ6FlCqcsuRh_TSqSPp6JXLN6TB
|
|
|
117
118
|
atomicshop/etws/const.py,sha256=v3x_IdCYeSKbCGywiZFOZln80ldpwKW5nuMDuUe51Jg,1257
|
|
118
119
|
atomicshop/etws/providers.py,sha256=CXNx8pYdjtpLIpA66IwrnE64XhY4U5ExnFBMLEb8Uzk,547
|
|
119
120
|
atomicshop/etws/sessions.py,sha256=b_KeiOvgOBJezJokN81TRlrvJiQNJlIWN4Z6UVjuxP0,1335
|
|
120
|
-
atomicshop/etws/trace.py,sha256=
|
|
121
|
+
atomicshop/etws/trace.py,sha256=p-5wegZ-5SuJ3UsprCCnlor8g2LPkF9ZOynLCYaEvXQ,7647
|
|
121
122
|
atomicshop/etws/traces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
122
|
-
atomicshop/etws/traces/trace_dns.py,sha256=
|
|
123
|
+
atomicshop/etws/traces/trace_dns.py,sha256=SWeex1mK7rrMzCmmL3qiGaD_jzbRiiwJ_mhMZz5TwJc,6879
|
|
123
124
|
atomicshop/etws/traces/trace_sysmon_process_creation.py,sha256=OM-bkK38uYMwWLZKNOTDa0Xdk3sO6sqsxoMUIiPvm5g,4656
|
|
125
|
+
atomicshop/etws/traces/trace_tcp.py,sha256=pQF4A_qM-HjEhzMQUyTmUqOWdvSaqqAg5PwnkVn2fy4,5236
|
|
124
126
|
atomicshop/file_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
125
127
|
atomicshop/file_io/csvs.py,sha256=zv0kKjRT-ZWRi0WpMIUQ_FKyP9Dt0f5Bc98Qsj6ClPU,9495
|
|
126
128
|
atomicshop/file_io/docxs.py,sha256=Nyt3hSpzwqUKZEP5p5efqNpjFs9XqkK40Kp7BbbPo7E,6245
|
|
@@ -168,7 +170,7 @@ atomicshop/permissions/ubuntu_permissions.py,sha256=n8z1vcIXDts4zLVue33dtJiTopjg
|
|
|
168
170
|
atomicshop/permissions/win_permissions.py,sha256=eDQm1jfK9x_hkbLqIJjFTwfqinAWQ0iSr0kW3XrF1BE,1272
|
|
169
171
|
atomicshop/process_poller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
170
172
|
atomicshop/process_poller/process_pool.py,sha256=4Qs427qd7OcBxu5PMFU5PTmyuxRy0vgj2GLsRt0IoEw,9565
|
|
171
|
-
atomicshop/process_poller/simple_process_pool.py,sha256=
|
|
173
|
+
atomicshop/process_poller/simple_process_pool.py,sha256=E0w66imLQQ0phHuTQfTaApOTurX0duE9vw1trjUY7rI,9894
|
|
172
174
|
atomicshop/process_poller/tracer_base.py,sha256=IOiHcnmF-MccOSCErixN5mve9RifZ9cPnGVHCIRchrs,1091
|
|
173
175
|
atomicshop/process_poller/pollers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
174
176
|
atomicshop/process_poller/pollers/psutil_pywin32wmi_dll.py,sha256=XRRfOIy62iOYU8IKRcyECWiL0rqQ35DeYbPsv_SHDVM,4510
|
|
@@ -191,7 +193,7 @@ atomicshop/wrappers/astw.py,sha256=VkYfkfyc_PJLIOxByT6L7B8uUmKY6-I8XGZl4t_z828,4
|
|
|
191
193
|
atomicshop/wrappers/configparserw.py,sha256=JwDTPjZoSrv44YKwIRcjyUnpN-FjgXVfMqMK_tJuSgU,22800
|
|
192
194
|
atomicshop/wrappers/cryptographyw.py,sha256=LfzTnwvJE03G6WZryOOf43VKhhnyMakzHpn8DPPCoy4,13252
|
|
193
195
|
atomicshop/wrappers/ffmpegw.py,sha256=wcq0ZnAe0yajBOuTKZCCaKI7CDBjkq7FAgdW5IsKcVE,6031
|
|
194
|
-
atomicshop/wrappers/githubw.py,sha256=
|
|
196
|
+
atomicshop/wrappers/githubw.py,sha256=vaVQg0pkHY63_TfIbCoWmHDPtI2rDPAqMYwsOl3GN78,22559
|
|
195
197
|
atomicshop/wrappers/msiw.py,sha256=GQLqud72nfex3kvO1bJSruNriCYTYX1_G1gSf1MPkIA,6118
|
|
196
198
|
atomicshop/wrappers/numpyw.py,sha256=sBV4gSKyr23kXTalqAb1oqttzE_2XxBooCui66jbAqc,1025
|
|
197
199
|
atomicshop/wrappers/olefilew.py,sha256=biD5m58rogifCYmYhJBrAFb9O_Bn_spLek_9HofLeYE,2051
|
|
@@ -327,8 +329,8 @@ atomicshop/wrappers/socketw/statistics_csv.py,sha256=fgMzDXI0cybwUEqAxprRmY3lqbh
|
|
|
327
329
|
atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
328
330
|
atomicshop/wrappers/winregw/winreg_installed_software.py,sha256=Qzmyktvob1qp6Tjk2DjLfAqr_yXV0sgWzdMW_9kwNjY,2345
|
|
329
331
|
atomicshop/wrappers/winregw/winreg_network.py,sha256=AENV88H1qDidrcpyM9OwEZxX5svfi-Jb4N6FkS1xtqA,8851
|
|
330
|
-
atomicshop-2.19.
|
|
331
|
-
atomicshop-2.19.
|
|
332
|
-
atomicshop-2.19.
|
|
333
|
-
atomicshop-2.19.
|
|
334
|
-
atomicshop-2.19.
|
|
332
|
+
atomicshop-2.19.12.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
|
|
333
|
+
atomicshop-2.19.12.dist-info/METADATA,sha256=IjBkWX6cc09byHYWcjX8hdW_eEmXddbKu1h0hNYUybM,10631
|
|
334
|
+
atomicshop-2.19.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
335
|
+
atomicshop-2.19.12.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
|
|
336
|
+
atomicshop-2.19.12.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|