maco 1.2.4__py3-none-any.whl → 1.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- demo_extractors/complex/complex.py +1 -2
- demo_extractors/elfy.py +1 -1
- demo_extractors/limit_other.py +10 -3
- demo_extractors/nothing.py +1 -1
- demo_extractors/requirements.txt +1 -0
- demo_extractors/shared.py +1 -0
- maco/base_test.py +15 -8
- maco/cli.py +8 -3
- maco/collector.py +11 -1
- maco/extractor.py +4 -4
- maco/model/__init__.py +1 -1
- maco/model/model.py +53 -19
- maco/utils.py +114 -103
- maco/yara.py +0 -1
- {maco-1.2.4.dist-info → maco-1.2.6.dist-info}/METADATA +14 -2
- maco-1.2.6.dist-info/RECORD +44 -0
- model_setup/maco/base_test.py +15 -8
- model_setup/maco/cli.py +8 -3
- model_setup/maco/collector.py +11 -1
- model_setup/maco/extractor.py +4 -4
- model_setup/maco/model/__init__.py +1 -1
- model_setup/maco/model/model.py +53 -19
- model_setup/maco/utils.py +114 -103
- model_setup/maco/yara.py +0 -1
- pipelines/test.yaml +21 -0
- tests/extractors/bob/bob.py +1 -4
- maco-1.2.4.dist-info/RECORD +0 -43
- {maco-1.2.4.dist-info → maco-1.2.6.dist-info}/LICENSE.md +0 -0
- {maco-1.2.4.dist-info → maco-1.2.6.dist-info}/WHEEL +0 -0
- {maco-1.2.4.dist-info → maco-1.2.6.dist-info}/entry_points.txt +0 -0
- {maco-1.2.4.dist-info → maco-1.2.6.dist-info}/top_level.txt +0 -0
model_setup/maco/model/model.py
CHANGED
|
@@ -59,31 +59,48 @@ class CategoryEnum(str, Enum):
|
|
|
59
59
|
# Malware related to an Advanced Persistent Threat (APT) group.
|
|
60
60
|
apt = "apt"
|
|
61
61
|
|
|
62
|
-
# A backdoor Trojan gives malicious users remote control over the infected computer.
|
|
62
|
+
# A backdoor Trojan gives malicious users remote control over the infected computer.
|
|
63
|
+
# They enable the author to do anything they wish on the infected computer including
|
|
64
|
+
# sending, receiving, launching and deleting files, displaying data and rebooting the computer.
|
|
65
|
+
# Backdoor Trojans are often used to unite a group of victim computers to form a botnet or
|
|
66
|
+
# zombie network that can be used for criminal purposes.
|
|
63
67
|
backdoor = "backdoor"
|
|
64
68
|
|
|
65
|
-
# Trojan Banker programs are designed to steal your account data for online banking systems,
|
|
69
|
+
# Trojan Banker programs are designed to steal your account data for online banking systems,
|
|
70
|
+
# e-payment systems and credit or debit cards.
|
|
66
71
|
banker = "banker"
|
|
67
72
|
|
|
68
|
-
# A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR)
|
|
73
|
+
# A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR)
|
|
74
|
+
# and Volume Boot Record (VBR).
|
|
69
75
|
bootkit = "bootkit"
|
|
70
76
|
|
|
71
|
-
# A malicious bot is self-propagating malware designed to infect a host and connect back to a central server
|
|
77
|
+
# A malicious bot is self-propagating malware designed to infect a host and connect back to a central server
|
|
78
|
+
# or servers that act as a command and control (C&C) center for an entire network of compromised devices,
|
|
79
|
+
# or botnet.
|
|
72
80
|
bot = "bot"
|
|
73
81
|
|
|
74
|
-
# A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without
|
|
82
|
+
# A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without
|
|
83
|
+
# the user's permission. The result is the placement of unwanted advertising into the browser,
|
|
84
|
+
# and possibly the replacement of an existing home page or search page with the hijacker page.
|
|
75
85
|
browser_hijacker = "browser_hijacker"
|
|
76
86
|
|
|
77
|
-
# Trojan bruteforcer are trying to brute force website in order to achieve something else
|
|
87
|
+
# Trojan bruteforcer are trying to brute force website in order to achieve something else
|
|
88
|
+
# (EX: Finding WordPress websites with default credentials).
|
|
78
89
|
bruteforcer = "bruteforcer"
|
|
79
90
|
|
|
80
|
-
# A type of trojan that can use your PC to 'click' on websites or applications.
|
|
91
|
+
# A type of trojan that can use your PC to 'click' on websites or applications.
|
|
92
|
+
# They are usually used to make money for a malicious hacker by clicking on online advertisements
|
|
93
|
+
# and making it look like the website gets more traffic than it does.
|
|
94
|
+
# They can also be used to skew online polls, install programs on your PC, or make unwanted software
|
|
95
|
+
# appear more popular than it is.
|
|
81
96
|
clickfraud = "clickfraud"
|
|
82
97
|
|
|
83
98
|
# Cryptocurrency mining malware.
|
|
84
99
|
cryptominer = "cryptominer"
|
|
85
100
|
|
|
86
|
-
# These programs conduct DoS (Denial of Service) attacks against a targeted web address.
|
|
101
|
+
# These programs conduct DoS (Denial of Service) attacks against a targeted web address.
|
|
102
|
+
# By sending multiple requests from your computer and several other infected computers,
|
|
103
|
+
# the attack can overwhelm the target address leading to a denial of service.
|
|
87
104
|
ddos = "ddos"
|
|
88
105
|
|
|
89
106
|
# Trojan Downloaders can download and install new versions of malicious programs in the target system.
|
|
@@ -92,49 +109,66 @@ class CategoryEnum(str, Enum):
|
|
|
92
109
|
# These programs are used by hackers in order to install malware or to prevent the detection of malicious programs.
|
|
93
110
|
dropper = "dropper"
|
|
94
111
|
|
|
95
|
-
# Exploit kits are programs that contain data or code that takes advantage of a vulnerability
|
|
112
|
+
# Exploit kits are programs that contain data or code that takes advantage of a vulnerability
|
|
113
|
+
# within an application that is running in the target system.
|
|
96
114
|
exploitkit = "exploitkit"
|
|
97
115
|
|
|
98
|
-
# Trojan FakeAV programs simulate the activity of antivirus software.
|
|
116
|
+
# Trojan FakeAV programs simulate the activity of antivirus software.
|
|
117
|
+
# They are designed to extort money in return for the detection and removal of threat, even though the
|
|
118
|
+
# threats that they report are actually non-existent.
|
|
99
119
|
fakeav = "fakeav"
|
|
100
120
|
|
|
101
121
|
# A type of tool that can be used to allow and maintain unauthorized access to your PC.
|
|
102
122
|
hacktool = "hacktool"
|
|
103
123
|
|
|
104
|
-
# A program that collects your personal information, such as your browsing history,
|
|
124
|
+
# A program that collects your personal information, such as your browsing history,
|
|
125
|
+
# and uses it without adequate consent.
|
|
105
126
|
infostealer = "infostealer"
|
|
106
127
|
|
|
107
|
-
# A keylogger monitors and logs every keystroke it can identify.
|
|
128
|
+
# A keylogger monitors and logs every keystroke it can identify.
|
|
129
|
+
# Once installed, the virus either keeps track of all the keys and stores the information locally,
|
|
130
|
+
# after which the hacker needs physical access to the computer to retrieve the information,
|
|
131
|
+
# or the logs are sent over the internet back to the hacker.
|
|
108
132
|
keylogger = "keylogger"
|
|
109
133
|
|
|
110
134
|
# A program that loads another application / memory space.
|
|
111
135
|
loader = "loader"
|
|
112
136
|
|
|
113
|
-
# A type of malware that hides its code and purpose to make it more difficult for
|
|
137
|
+
# A type of malware that hides its code and purpose to make it more difficult for
|
|
138
|
+
# security software to detect or remove it.
|
|
114
139
|
obfuscator = "obfuscator"
|
|
115
140
|
|
|
116
|
-
# Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS)
|
|
141
|
+
# Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS)
|
|
142
|
+
# and payment terminals with the intent to obtain credit card and debit card information.
|
|
117
143
|
pos = "pos"
|
|
118
144
|
|
|
119
|
-
# This type of trojan allows unauthorized parties to use the infected computer as a proxy server
|
|
145
|
+
# This type of trojan allows unauthorized parties to use the infected computer as a proxy server
|
|
146
|
+
# to access the Internet anonymously.
|
|
120
147
|
proxy = "proxy"
|
|
121
148
|
|
|
122
149
|
# A program that can be used by a remote hacker to gain access and control of an infected machine.
|
|
123
150
|
rat = "rat"
|
|
124
151
|
|
|
125
|
-
# This type of malware can modify data in the target computer so the operating system
|
|
152
|
+
# This type of malware can modify data in the target computer so the operating system
|
|
153
|
+
# will stop running correctly or the data is no longer accessible.
|
|
154
|
+
# The criminal will only restore the computer state or data after a ransom is paid to them
|
|
155
|
+
# (mostly using cryptocurrency).
|
|
126
156
|
ransomware = "ransomware"
|
|
127
157
|
|
|
128
158
|
# A reverse proxy is a server that receives requests from the internet and forwards them to a small set of servers.
|
|
129
159
|
reverse_proxy = "reverse_proxy"
|
|
130
160
|
|
|
131
|
-
# Rootkits are designed to conceal certain objects or activities in the system.
|
|
161
|
+
# Rootkits are designed to conceal certain objects or activities in the system.
|
|
162
|
+
# Often their main purpose is to prevent malicious programs being detected
|
|
163
|
+
# in order to extend the period in which programs can run on an infected computer.
|
|
132
164
|
rootkit = "rootkit"
|
|
133
165
|
|
|
134
|
-
# This type of malware scan the internet / network(s) / system(s) / service(s) to collect information.
|
|
166
|
+
# This type of malware scan the internet / network(s) / system(s) / service(s) to collect information.
|
|
167
|
+
# That information could be used later to perpetuate an cyber attack.
|
|
135
168
|
scanner = "scanner"
|
|
136
169
|
|
|
137
|
-
# Scareware is a form of malware which uses social engineering to cause shock, anxiety,
|
|
170
|
+
# Scareware is a form of malware which uses social engineering to cause shock, anxiety,
|
|
171
|
+
# or the perception of a threat in order to manipulate users into buying unwanted software.
|
|
138
172
|
scareware = "scareware"
|
|
139
173
|
|
|
140
174
|
# Malware that is sending spam.
|
model_setup/maco/utils.py
CHANGED
|
@@ -4,14 +4,14 @@ import importlib.machinery
|
|
|
4
4
|
import importlib.util
|
|
5
5
|
import inspect
|
|
6
6
|
import json
|
|
7
|
+
import logging
|
|
7
8
|
import logging.handlers
|
|
9
|
+
import multiprocessing
|
|
8
10
|
import os
|
|
9
11
|
import re
|
|
10
12
|
import shutil
|
|
11
13
|
import subprocess
|
|
12
14
|
import sys
|
|
13
|
-
import multiprocessing
|
|
14
|
-
import logging
|
|
15
15
|
import tempfile
|
|
16
16
|
|
|
17
17
|
from maco import yara
|
|
@@ -27,8 +27,11 @@ from glob import glob
|
|
|
27
27
|
from logging import Logger
|
|
28
28
|
from pkgutil import walk_packages
|
|
29
29
|
from types import ModuleType
|
|
30
|
-
from typing import Callable, Dict, List, Set, Tuple
|
|
30
|
+
from typing import Callable, Dict, List, Set, Tuple, Union
|
|
31
|
+
|
|
32
|
+
from uv import find_uv_bin
|
|
31
33
|
|
|
34
|
+
from maco import model
|
|
32
35
|
from maco.extractor import Extractor
|
|
33
36
|
|
|
34
37
|
logger = logging.getLogger("maco.lib.utils")
|
|
@@ -38,22 +41,10 @@ VENV_DIRECTORY_NAME = ".venv"
|
|
|
38
41
|
RELATIVE_FROM_RE = re.compile(r"from (\.+)")
|
|
39
42
|
RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
|
|
40
43
|
|
|
41
|
-
|
|
42
|
-
# Attempt to use the uv package manager (Recommended)
|
|
43
|
-
from uv import find_uv_bin
|
|
44
|
+
UV_BIN = find_uv_bin()
|
|
44
45
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
PIP_CMD = f"{UV_BIN} pip"
|
|
48
|
-
VENV_CREATE_CMD = f"{UV_BIN} venv"
|
|
49
|
-
PACKAGE_MANAGER = "uv"
|
|
50
|
-
except ImportError:
|
|
51
|
-
# Otherwise default to pip
|
|
52
|
-
from sys import executable
|
|
53
|
-
|
|
54
|
-
PIP_CMD = "pip"
|
|
55
|
-
VENV_CREATE_CMD = f"{executable} -m venv"
|
|
56
|
-
PACKAGE_MANAGER = "pip"
|
|
46
|
+
PIP_CMD = f"{UV_BIN} pip"
|
|
47
|
+
VENV_CREATE_CMD = f"{UV_BIN} venv"
|
|
57
48
|
|
|
58
49
|
|
|
59
50
|
class Base64Decoder(json.JSONDecoder):
|
|
@@ -189,11 +180,9 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
|
|
|
189
180
|
for pattern in [RELATIVE_FROM_IMPORT_RE, RELATIVE_FROM_RE]:
|
|
190
181
|
for match in pattern.findall(data):
|
|
191
182
|
depth = match.count(".")
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
1,
|
|
196
|
-
)
|
|
183
|
+
abspath = ".".join(split[depth - 1 : split.index(package) + 1][::-1])
|
|
184
|
+
abspath += "." if pattern == RELATIVE_FROM_RE else ""
|
|
185
|
+
data = data.replace(f"from {match}", f"from {abspath}", 1)
|
|
197
186
|
f.write(data)
|
|
198
187
|
|
|
199
188
|
if scanner.match(path):
|
|
@@ -210,9 +199,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
|
|
|
210
199
|
return extractor_dirs, extractor_files
|
|
211
200
|
|
|
212
201
|
|
|
213
|
-
def
|
|
202
|
+
def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
|
|
214
203
|
venvs = []
|
|
215
|
-
logger.info("Creating virtual environment(s)..")
|
|
216
204
|
env = deepcopy(os.environ)
|
|
217
205
|
stop_directory = os.path.dirname(sorted(directories)[0])
|
|
218
206
|
# Track directories that we've already visited
|
|
@@ -222,17 +210,23 @@ def create_virtual_environments(directories: List[str], python_version: str, log
|
|
|
222
210
|
while dir != stop_directory and dir not in visited_dirs:
|
|
223
211
|
req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
|
|
224
212
|
if req_files:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
213
|
+
# create a virtual environment, otherwise directly install into current env
|
|
214
|
+
if create_venv:
|
|
215
|
+
venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
|
|
216
|
+
logger.info(f"Updating virtual environment {venv_path}")
|
|
217
|
+
env.update({"VIRTUAL_ENV": venv_path})
|
|
218
|
+
# Create a virtual environment for the directory
|
|
219
|
+
if not os.path.exists(venv_path):
|
|
220
|
+
cmd = f"{VENV_CREATE_CMD} --python {python_version}"
|
|
221
|
+
subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
|
|
233
222
|
|
|
234
223
|
# Install/Update the packages in the environment
|
|
235
|
-
install_command = PIP_CMD.split(" ") + ["install"
|
|
224
|
+
install_command = PIP_CMD.split(" ") + ["install"]
|
|
225
|
+
# When running locally, only install packages to required spec.
|
|
226
|
+
# This prevents issues during maco development and building extractors against local libraries.
|
|
227
|
+
if create_venv:
|
|
228
|
+
# when running in custom virtual environment, always upgrade packages.
|
|
229
|
+
install_command.append("-U")
|
|
236
230
|
|
|
237
231
|
# Update the pip install command depending on where the dependencies are coming from
|
|
238
232
|
if "requirements.txt" in req_files:
|
|
@@ -253,7 +247,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
|
|
|
253
247
|
|
|
254
248
|
install_command.extend(pyproject_command)
|
|
255
249
|
|
|
250
|
+
# always require maco to be installed
|
|
251
|
+
install_command.append("maco")
|
|
256
252
|
logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
|
|
253
|
+
# this uses VIRTUAL_ENV to control usage of a virtual environment
|
|
257
254
|
p = subprocess.run(
|
|
258
255
|
install_command,
|
|
259
256
|
cwd=dir,
|
|
@@ -264,10 +261,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
|
|
|
264
261
|
if b"is being installed using the legacy" in p.stderr:
|
|
265
262
|
# Ignore these types of errors
|
|
266
263
|
continue
|
|
267
|
-
logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
|
|
264
|
+
logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
|
|
268
265
|
else:
|
|
269
|
-
logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
|
|
270
|
-
|
|
266
|
+
logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
|
|
267
|
+
if create_venv:
|
|
268
|
+
venvs.append(venv_path)
|
|
271
269
|
|
|
272
270
|
# Cleanup any build directories that are the product of package installation
|
|
273
271
|
expected_build_path = os.path.join(dir, "build")
|
|
@@ -311,10 +309,12 @@ def register_extractors(
|
|
|
311
309
|
):
|
|
312
310
|
package_name = os.path.basename(current_directory)
|
|
313
311
|
parent_directory = os.path.dirname(current_directory)
|
|
314
|
-
if package_name in sys.modules:
|
|
312
|
+
if venvs and package_name in sys.modules:
|
|
315
313
|
# this may happen as part of testing if some part of the extractor code was directly imported
|
|
316
|
-
logger.warning(
|
|
317
|
-
|
|
314
|
+
logger.warning(
|
|
315
|
+
f"Looks like {package_name} is already loaded. "
|
|
316
|
+
"If your maco extractor overlaps an existing package name this could cause problems."
|
|
317
|
+
)
|
|
318
318
|
|
|
319
319
|
try:
|
|
320
320
|
# Modify the PATH so we can recognize this new package on import
|
|
@@ -389,6 +389,7 @@ def register_extractors(
|
|
|
389
389
|
# We were able to find all the extractor files
|
|
390
390
|
break
|
|
391
391
|
|
|
392
|
+
|
|
392
393
|
def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
|
|
393
394
|
"""Ensures logging is set up correctly for a child process and then executes the callback."""
|
|
394
395
|
logger = logging.getLogger()
|
|
@@ -397,37 +398,33 @@ def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType,
|
|
|
397
398
|
logger.addHandler(qh)
|
|
398
399
|
callback(*args, **kwargs, logger=logger)
|
|
399
400
|
|
|
401
|
+
|
|
400
402
|
def import_extractors(
|
|
401
403
|
extractor_module_callback: Callable[[ModuleType, str], bool],
|
|
402
404
|
*,
|
|
403
405
|
root_directory: str,
|
|
404
406
|
scanner: yara.Rules,
|
|
405
|
-
create_venv: bool
|
|
406
|
-
python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
|
|
407
|
+
create_venv: bool,
|
|
407
408
|
logger: Logger,
|
|
409
|
+
python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
|
|
408
410
|
):
|
|
409
411
|
extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
|
|
410
412
|
|
|
411
413
|
logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
|
|
412
414
|
logger.debug(extractor_files)
|
|
413
415
|
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
venvs = create_virtual_environments(extractor_dirs, python_version, logger)
|
|
417
|
-
else:
|
|
418
|
-
# Look for pre-existing virtual environments, if any
|
|
419
|
-
logger.info("Checking for pre-existing virtual environment(s)..")
|
|
420
|
-
venvs = [
|
|
421
|
-
os.path.join(root, VENV_DIRECTORY_NAME)
|
|
422
|
-
for root, dirs, _ in os.walk(root_directory)
|
|
423
|
-
if VENV_DIRECTORY_NAME in dirs
|
|
424
|
-
]
|
|
416
|
+
# Install packages into the current environment or dynamically created virtual environments
|
|
417
|
+
venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
|
|
425
418
|
|
|
426
419
|
# With the environment prepared, we can now hunt for the extractors and register them
|
|
427
420
|
logger.info("Registering extractors..")
|
|
428
421
|
register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
|
|
429
422
|
|
|
430
423
|
|
|
424
|
+
# holds cached extractors when not running in venv mode
|
|
425
|
+
_loaded_extractors: Dict[str, Extractor] = {}
|
|
426
|
+
|
|
427
|
+
|
|
431
428
|
def run_extractor(
|
|
432
429
|
sample_path,
|
|
433
430
|
module_name,
|
|
@@ -436,55 +433,69 @@ def run_extractor(
|
|
|
436
433
|
venv,
|
|
437
434
|
venv_script=VENV_SCRIPT,
|
|
438
435
|
json_decoder=Base64Decoder,
|
|
439
|
-
) -> Dict[str, dict]:
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
436
|
+
) -> Union[Dict[str, dict], model.ExtractorModel]:
|
|
437
|
+
"""Runs the maco extractor against sample either in current process or child process."""
|
|
438
|
+
if not venv:
|
|
439
|
+
key = f"{module_name}_{extractor_class}"
|
|
440
|
+
if key not in _loaded_extractors:
|
|
441
|
+
# dynamic import of extractor
|
|
442
|
+
mod = importlib.import_module(module_name)
|
|
443
|
+
extractor_cls = mod.__getattribute__(extractor_class)
|
|
444
|
+
extractor = extractor_cls()
|
|
445
|
+
else:
|
|
446
|
+
# retrieve cached extractor
|
|
447
|
+
extractor = _loaded_extractors[key]
|
|
448
|
+
if extractor.yara_compiled:
|
|
449
|
+
matches = extractor.yara_compiled.match(sample_path)
|
|
450
|
+
loaded = extractor.run(open(sample_path, "rb"), matches=matches)
|
|
451
|
+
else:
|
|
452
|
+
# execute extractor in child process with separate virtual environment
|
|
453
|
+
# Write temporary script in the same directory as extractor to resolve relative imports
|
|
444
454
|
python_exe = os.path.join(venv, "bin", "python")
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
455
|
+
dirname = os.path.dirname(module_path)
|
|
456
|
+
with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
|
|
457
|
+
with tempfile.NamedTemporaryFile() as output:
|
|
458
|
+
parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
|
|
459
|
+
root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
|
|
460
|
+
|
|
461
|
+
script.write(
|
|
462
|
+
venv_script.format(
|
|
463
|
+
parent_package_path=parent_package_path,
|
|
464
|
+
module_name=module_name,
|
|
465
|
+
module_class=extractor_class,
|
|
466
|
+
sample_path=sample_path,
|
|
467
|
+
output_path=output.name,
|
|
468
|
+
)
|
|
458
469
|
)
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
470
|
+
script.flush()
|
|
471
|
+
cwd = root_directory
|
|
472
|
+
custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
|
|
473
|
+
|
|
474
|
+
if custom_module.startswith("src."):
|
|
475
|
+
# src layout found, which means the actual module content is within 'src' directory
|
|
476
|
+
custom_module = custom_module[4:]
|
|
477
|
+
cwd = os.path.join(cwd, "src")
|
|
478
|
+
|
|
479
|
+
# run the maco extractor in full venv process isolation (slow)
|
|
480
|
+
proc = subprocess.run(
|
|
481
|
+
[python_exe, "-m", custom_module],
|
|
482
|
+
cwd=cwd,
|
|
483
|
+
capture_output=True,
|
|
484
|
+
)
|
|
485
|
+
stderr = proc.stderr.decode()
|
|
486
|
+
try:
|
|
487
|
+
# Load results and return them
|
|
488
|
+
output.seek(0)
|
|
489
|
+
loaded = json.load(output, cls=json_decoder)
|
|
490
|
+
except Exception as e:
|
|
491
|
+
# If there was an error raised during runtime, then propagate
|
|
492
|
+
delim = f'File "{module_path}"'
|
|
493
|
+
exception = stderr
|
|
494
|
+
if delim in exception:
|
|
495
|
+
exception = f"{delim}{exception.split(delim, 1)[1]}"
|
|
496
|
+
# print extractor logging at error level
|
|
497
|
+
logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
|
|
498
|
+
raise Exception(exception) from e
|
|
499
|
+
# ensure that extractor logging is available
|
|
500
|
+
logger.info(f"maco extractor stderr:\n{stderr}")
|
|
501
|
+
return loaded
|
model_setup/maco/yara.py
CHANGED
pipelines/test.yaml
CHANGED
|
@@ -7,6 +7,27 @@ pool:
|
|
|
7
7
|
vmImage: "ubuntu-22.04"
|
|
8
8
|
|
|
9
9
|
jobs:
|
|
10
|
+
- job: style_test
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
Python3_12:
|
|
14
|
+
python.version: "3.12"
|
|
15
|
+
timeoutInMinutes: 10
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- task: UsePythonVersion@0
|
|
19
|
+
displayName: Set python version
|
|
20
|
+
inputs:
|
|
21
|
+
versionSpec: "$(python.version)"
|
|
22
|
+
|
|
23
|
+
- script: |
|
|
24
|
+
python -m pip install -U tox
|
|
25
|
+
displayName: Install tox
|
|
26
|
+
|
|
27
|
+
- script: |
|
|
28
|
+
python -m tox -e style
|
|
29
|
+
displayName: "Run style tests"
|
|
30
|
+
|
|
10
31
|
- job: run_test
|
|
11
32
|
strategy:
|
|
12
33
|
matrix:
|
tests/extractors/bob/bob.py
CHANGED
maco-1.2.4.dist-info/RECORD
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
demo_extractors/elfy.py,sha256=AAFr5i1aivPwO4nycyXJEud57EpVNA-5k_2GicWesbY,771
|
|
3
|
-
demo_extractors/limit_other.py,sha256=RAFx_0K_WnhUURA5uwXGmjrWODrAuLZFBxqZcWaxf64,944
|
|
4
|
-
demo_extractors/nothing.py,sha256=3aeQJTY-dakmVXmyfmrRM8YCQVT7q3bq880DFH1Ol_Y,607
|
|
5
|
-
demo_extractors/shared.py,sha256=Wlvy77SCAR97gxi8uUhGYyjxGmDb-pOSvN8b1rXrVWs,304
|
|
6
|
-
demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
demo_extractors/complex/complex.py,sha256=JFKqBGKwkuDSz4zZUJuqhCLUQv6dlCMhBqNj33grBsE,2323
|
|
8
|
-
demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
|
|
9
|
-
maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
|
|
11
|
-
maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
|
|
12
|
-
maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
|
|
13
|
-
maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
|
|
14
|
-
maco/utils.py,sha256=vQeJKw4whWTXp1mTd2oEhfvL4nvvgAzWjBnCp2XxWLI,19275
|
|
15
|
-
maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
|
|
16
|
-
maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
|
|
17
|
-
maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
|
|
18
|
-
model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
model_setup/maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
|
|
20
|
-
model_setup/maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
|
|
21
|
-
model_setup/maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
|
|
22
|
-
model_setup/maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
|
|
23
|
-
model_setup/maco/utils.py,sha256=vQeJKw4whWTXp1mTd2oEhfvL4nvvgAzWjBnCp2XxWLI,19275
|
|
24
|
-
model_setup/maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
|
|
25
|
-
model_setup/maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
|
|
26
|
-
model_setup/maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
|
|
27
|
-
pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
|
|
28
|
-
pipelines/test.yaml,sha256=3KOoo-8SqP_bTAscsz5V3xxnuL91J-62mTjnQD1Btag,1019
|
|
29
|
-
tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
|
|
30
|
-
tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
|
|
31
|
-
tests/data/trigger_complex.txt.cart,sha256=Z7qF1Zi640O45Znkl9ooP2RhSLAEqY0NRf51d-q7utU,345
|
|
32
|
-
tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
|
-
tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
|
|
34
|
-
tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
|
|
35
|
-
tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
|
|
36
|
-
tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
37
|
-
tests/extractors/bob/bob.py,sha256=Gy5p8KssJX87cwa9vVv8UBODF_ulbUteZXh15frW2hs,247
|
|
38
|
-
maco-1.2.4.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
|
|
39
|
-
maco-1.2.4.dist-info/METADATA,sha256=EoAEnCfbaXe8eUFAMHj3-C9lQIDR9Ss31NcY4CbEXjI,15610
|
|
40
|
-
maco-1.2.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
41
|
-
maco-1.2.4.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
|
|
42
|
-
maco-1.2.4.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
|
|
43
|
-
maco-1.2.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|