assemblyline-v4-service 4.4.0.24__py3-none-any.whl → 4.4.0.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of assemblyline-v4-service might be problematic. Click here for more details.
- assemblyline_v4_service/VERSION +1 -1
- assemblyline_v4_service/common/api.py +3 -2
- assemblyline_v4_service/common/base.py +3 -4
- assemblyline_v4_service/common/helper.py +1 -2
- assemblyline_v4_service/common/{extractor/ocr.py → ocr.py} +0 -1
- assemblyline_v4_service/common/ontology_helper.py +7 -8
- assemblyline_v4_service/common/request.py +4 -5
- assemblyline_v4_service/common/result.py +3 -3
- assemblyline_v4_service/common/task.py +3 -3
- assemblyline_v4_service/common/utils.py +2 -2
- assemblyline_v4_service/updater/helper.py +4 -0
- {assemblyline_v4_service-4.4.0.24.dist-info → assemblyline_v4_service-4.4.0.26.dist-info}/METADATA +1 -1
- assemblyline_v4_service-4.4.0.26.dist-info/RECORD +28 -0
- assemblyline_v4_service/common/balbuzard/__init__.py +0 -0
- assemblyline_v4_service/common/balbuzard/balbuzard.py +0 -656
- assemblyline_v4_service/common/balbuzard/bbcrack.py +0 -830
- assemblyline_v4_service/common/balbuzard/patterns.py +0 -650
- assemblyline_v4_service/common/dynamic_service_helper.py +0 -3631
- assemblyline_v4_service/common/extractor/__init__.py +0 -1
- assemblyline_v4_service/common/extractor/base64.py +0 -86
- assemblyline_v4_service/common/extractor/pe_file.py +0 -51
- assemblyline_v4_service/common/icap.py +0 -149
- assemblyline_v4_service/common/keytool_parse.py +0 -66
- assemblyline_v4_service/common/pestudio/__init__.py +0 -0
- assemblyline_v4_service/common/pestudio/xml/__init__.py +0 -0
- assemblyline_v4_service/common/pestudio/xml/features.xml +0 -5607
- assemblyline_v4_service/common/pestudio/xml/functions.xml +0 -5824
- assemblyline_v4_service/common/pestudio/xml/languages.xml +0 -375
- assemblyline_v4_service/common/pestudio/xml/resources.xml +0 -511
- assemblyline_v4_service/common/pestudio/xml/signatures.xml +0 -29105
- assemblyline_v4_service/common/pestudio/xml/strings.xml +0 -2379
- assemblyline_v4_service/common/safelist_helper.py +0 -73
- assemblyline_v4_service/common/section_reducer.py +0 -43
- assemblyline_v4_service/common/tag_helper.py +0 -117
- assemblyline_v4_service/common/tag_reducer.py +0 -242
- assemblyline_v4_service/testing/__init__.py +0 -0
- assemblyline_v4_service/testing/helper.py +0 -463
- assemblyline_v4_service/testing/regenerate_results.py +0 -37
- assemblyline_v4_service-4.4.0.24.dist-info/RECORD +0 -53
- {assemblyline_v4_service-4.4.0.24.dist-info → assemblyline_v4_service-4.4.0.26.dist-info}/LICENCE.md +0 -0
- {assemblyline_v4_service-4.4.0.24.dist-info → assemblyline_v4_service-4.4.0.26.dist-info}/WHEEL +0 -0
- {assemblyline_v4_service-4.4.0.24.dist-info → assemblyline_v4_service-4.4.0.26.dist-info}/top_level.txt +0 -0
|
@@ -1,463 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import os
|
|
3
|
-
import pytest
|
|
4
|
-
import shutil
|
|
5
|
-
import re
|
|
6
|
-
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
from assemblyline.common import forge
|
|
10
|
-
from assemblyline.common.dict_utils import flatten
|
|
11
|
-
from assemblyline.common.uid import get_random_id
|
|
12
|
-
from assemblyline.odm.messages.task import Task as ServiceTask
|
|
13
|
-
from assemblyline_v4_service.common import helper
|
|
14
|
-
from assemblyline_v4_service.common.request import ServiceRequest
|
|
15
|
-
from assemblyline_v4_service.common.task import Task
|
|
16
|
-
from cart import unpack_file
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class FileMissing(Exception):
|
|
20
|
-
pass
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class HeuristicFiletypeMismatch(Exception):
|
|
24
|
-
pass
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class IssueHelper:
|
|
28
|
-
ACTION_MISSING = "--"
|
|
29
|
-
ACTION_ADDED = "++"
|
|
30
|
-
ACTION_CHANGED = "-+"
|
|
31
|
-
|
|
32
|
-
TYPE_HEUR = "HEURISTICS"
|
|
33
|
-
TYPE_TAGS = "TAGS"
|
|
34
|
-
TYPE_TEMP = "TEMP_DATA"
|
|
35
|
-
TYPE_SUPPLEMENTARY = "SUPPLEMENTARY"
|
|
36
|
-
TYPE_EXTRACTED = "EXTRACTED"
|
|
37
|
-
TYPE_EXTRA = "EXTRA"
|
|
38
|
-
|
|
39
|
-
def __init__(self):
|
|
40
|
-
self.issues = {}
|
|
41
|
-
|
|
42
|
-
def add_issue(self, itype: str, action: str, message: str):
|
|
43
|
-
self.issues.setdefault(itype, [])
|
|
44
|
-
self.issues[itype].append((action, message))
|
|
45
|
-
|
|
46
|
-
def get_issue_list(self):
|
|
47
|
-
return [f"[{k.upper()}] {action.capitalize()} {message}"
|
|
48
|
-
for k, v in self.issues.items()
|
|
49
|
-
for action, message in v]
|
|
50
|
-
|
|
51
|
-
def get_issues(self):
|
|
52
|
-
return self.issues
|
|
53
|
-
|
|
54
|
-
def has_issues(self):
|
|
55
|
-
return len(self.issues.keys()) != 0
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
class TestHelper:
|
|
59
|
-
def __init__(self, service_class, result_folder, extra_sample_locations=None):
|
|
60
|
-
# Set service class to use
|
|
61
|
-
self.service_class = service_class
|
|
62
|
-
|
|
63
|
-
# Set location for samples
|
|
64
|
-
self.locations = []
|
|
65
|
-
|
|
66
|
-
# Extra samples location
|
|
67
|
-
if extra_sample_locations:
|
|
68
|
-
self.locations.append(extra_sample_locations)
|
|
69
|
-
|
|
70
|
-
# Main samples location
|
|
71
|
-
full_samples_location = os.environ.get("FULL_SAMPLES_LOCATION", None)
|
|
72
|
-
if full_samples_location:
|
|
73
|
-
self.locations.append(full_samples_location)
|
|
74
|
-
|
|
75
|
-
# Set result folder location
|
|
76
|
-
self.result_folder = result_folder
|
|
77
|
-
|
|
78
|
-
# Load identify
|
|
79
|
-
self.identify = forge.get_identify(use_cache=False)
|
|
80
|
-
|
|
81
|
-
# Load submission params
|
|
82
|
-
self.submission_params = helper.get_service_attributes().submission_params
|
|
83
|
-
|
|
84
|
-
# Load service heuristic
|
|
85
|
-
self.heuristics = helper.get_heuristics()
|
|
86
|
-
|
|
87
|
-
def _create_service_task(self, file_path, params):
|
|
88
|
-
fileinfo_keys = ["magic", "md5", "mime", "sha1", "sha256", "size", "type"]
|
|
89
|
-
|
|
90
|
-
# Set proper default values
|
|
91
|
-
if params is None:
|
|
92
|
-
params = {}
|
|
93
|
-
|
|
94
|
-
metadata = params.get('metadata', {})
|
|
95
|
-
temp_submission_data = params.get('temp_submission_data', {})
|
|
96
|
-
submission_params = params.get('submission_params', {})
|
|
97
|
-
tags = params.get('tags', [])
|
|
98
|
-
filename = params.get('filename', os.path.basename(file_path))
|
|
99
|
-
|
|
100
|
-
return ServiceTask(
|
|
101
|
-
{
|
|
102
|
-
"sid": get_random_id(),
|
|
103
|
-
"metadata": metadata,
|
|
104
|
-
"deep_scan": False,
|
|
105
|
-
"service_name": self.service_class.__name__,
|
|
106
|
-
"service_config": {param.name: submission_params.get(param.name, param.default)
|
|
107
|
-
for param in self.submission_params},
|
|
108
|
-
"fileinfo": {k: v for k, v in self.identify.fileinfo(file_path).items() if k in fileinfo_keys},
|
|
109
|
-
"filename": filename,
|
|
110
|
-
"min_classification": "TLP:C",
|
|
111
|
-
"max_files": 501,
|
|
112
|
-
"ttl": 3600,
|
|
113
|
-
"temporary_submission_data": [
|
|
114
|
-
{'name': name, 'value': value} for name, value in temp_submission_data.items()
|
|
115
|
-
],
|
|
116
|
-
"tags": tags,
|
|
117
|
-
}
|
|
118
|
-
)
|
|
119
|
-
|
|
120
|
-
def _find_sample(self, sample):
|
|
121
|
-
# Assume samples are carted
|
|
122
|
-
sample = f"{sample.split('_', 1)[0]}.cart"
|
|
123
|
-
|
|
124
|
-
for location in self.locations:
|
|
125
|
-
p = [path for path in Path(location).rglob(sample)]
|
|
126
|
-
if len(p) == 1:
|
|
127
|
-
return p[0]
|
|
128
|
-
|
|
129
|
-
raise FileMissing(sample)
|
|
130
|
-
|
|
131
|
-
@staticmethod
|
|
132
|
-
def _generalize_result(result, temp_submission_data=None):
|
|
133
|
-
# Create a result for this file that contain generalized information for testing and
|
|
134
|
-
# detailed information as well so the service writter has a better idea of the impact
|
|
135
|
-
# of its changes to the service output.
|
|
136
|
-
generalized_results = {
|
|
137
|
-
"files": {
|
|
138
|
-
"extracted": sorted(
|
|
139
|
-
[{"name": x["name"], "sha256": x["sha256"]}
|
|
140
|
-
for x in result.get("response", {}).get("extracted", [])], key=lambda x: x["sha256"]
|
|
141
|
-
),
|
|
142
|
-
"supplementary": sorted(
|
|
143
|
-
[{"name": x["name"], "sha256": x["sha256"]}
|
|
144
|
-
for x in result.get("response", {}).get("supplementary", [])], key=lambda x: x["sha256"]
|
|
145
|
-
)
|
|
146
|
-
},
|
|
147
|
-
"results": {
|
|
148
|
-
"heuristics": [],
|
|
149
|
-
"tags": {},
|
|
150
|
-
"temp_submission_data": temp_submission_data
|
|
151
|
-
},
|
|
152
|
-
"extra": {
|
|
153
|
-
"sections": [],
|
|
154
|
-
"score": result.get("result", {}).get('score', 0),
|
|
155
|
-
"drop_file": result.get("drop", False)
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
# Parse sections
|
|
160
|
-
for section in result.get("result", {}).get("sections", []):
|
|
161
|
-
|
|
162
|
-
# Add section to extras (This will not be tested)
|
|
163
|
-
generalized_results['extra']['sections'].append(section)
|
|
164
|
-
|
|
165
|
-
# Parse Heuristics
|
|
166
|
-
heuristic = section.get('heuristic', None)
|
|
167
|
-
sigs = []
|
|
168
|
-
heur_id = None
|
|
169
|
-
if heuristic:
|
|
170
|
-
sigs = list(heuristic['signatures'].keys())
|
|
171
|
-
heur_id = heuristic['heur_id']
|
|
172
|
-
generalized_results["results"]["heuristics"].append(
|
|
173
|
-
{
|
|
174
|
-
"heur_id": heur_id,
|
|
175
|
-
"attack_ids": heuristic['attack_ids'],
|
|
176
|
-
'signatures': sigs
|
|
177
|
-
}
|
|
178
|
-
)
|
|
179
|
-
# Sort Heuristics
|
|
180
|
-
generalized_results["results"]["heuristics"] = \
|
|
181
|
-
sorted(generalized_results["results"]["heuristics"], key=lambda x: x["heur_id"])
|
|
182
|
-
|
|
183
|
-
# Parse tags
|
|
184
|
-
for k, v in flatten(section.get("tags", {})).items():
|
|
185
|
-
generalized_results["results"]["tags"].setdefault(k, [])
|
|
186
|
-
for tag in v:
|
|
187
|
-
generalized_results["results"]["tags"][k].append({
|
|
188
|
-
"value": tag,
|
|
189
|
-
"heur_id": heur_id,
|
|
190
|
-
"signatures": sigs
|
|
191
|
-
})
|
|
192
|
-
# Sort Tags
|
|
193
|
-
for k, v in generalized_results["results"]["tags"].items():
|
|
194
|
-
try:
|
|
195
|
-
generalized_results["results"]["tags"][k] = sorted(v, key=lambda x: x["value"])
|
|
196
|
-
except TypeError:
|
|
197
|
-
# Sorting for list with different types: https://stackoverflow.com/a/68416981
|
|
198
|
-
type_weights = {}
|
|
199
|
-
for element in v:
|
|
200
|
-
if type(element["value"]) not in type_weights:
|
|
201
|
-
type_weights[type(element["value"])] = len(type_weights)
|
|
202
|
-
|
|
203
|
-
generalized_results["results"]["tags"][k] = sorted(
|
|
204
|
-
v, key=lambda x: (type_weights[type(x["value"])], str(x["value"]))
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
return generalized_results
|
|
208
|
-
|
|
209
|
-
def _execute_sample(self, sample, save=False, save_files=False):
|
|
210
|
-
file_path = os.path.join("/tmp", sample.split('_', 1)[0])
|
|
211
|
-
cls = None
|
|
212
|
-
|
|
213
|
-
try:
|
|
214
|
-
# Find and unpack sample
|
|
215
|
-
sample_path = self._find_sample(sample)
|
|
216
|
-
unpack_file(sample_path, file_path)
|
|
217
|
-
|
|
218
|
-
# Load optional submission parameters
|
|
219
|
-
params_file = os.path.join(self.result_folder, sample, 'params.json')
|
|
220
|
-
if os.path.exists(params_file):
|
|
221
|
-
params = json.load(open(params_file))
|
|
222
|
-
else:
|
|
223
|
-
params = {}
|
|
224
|
-
|
|
225
|
-
# Initialize service class
|
|
226
|
-
cls = self.service_class(params.get('config', {}))
|
|
227
|
-
cls.start()
|
|
228
|
-
|
|
229
|
-
# Create the service request
|
|
230
|
-
task = Task(self._create_service_task(file_path, params))
|
|
231
|
-
service_request = ServiceRequest(task)
|
|
232
|
-
cls._working_directory = task.working_directory
|
|
233
|
-
|
|
234
|
-
# Execute the service
|
|
235
|
-
cls.execute(service_request)
|
|
236
|
-
|
|
237
|
-
# Get results from the scan
|
|
238
|
-
results = self._generalize_result(task.get_service_result(), task.temp_submission_data)
|
|
239
|
-
|
|
240
|
-
# Save results if needs be
|
|
241
|
-
if save:
|
|
242
|
-
# If we are re-writing the results, validate that the heuristics raised were meant for the sample
|
|
243
|
-
for heuristic in results["results"]["heuristics"]:
|
|
244
|
-
if not re.match(self.heuristics[heuristic["heur_id"]].filetype, task.file_type):
|
|
245
|
-
raise HeuristicFiletypeMismatch(
|
|
246
|
-
(
|
|
247
|
-
f"Tried to raise Heuristic {heuristic['heur_id']} "
|
|
248
|
-
f"({self.heuristics[heuristic['heur_id']].filetype}) for filetype {task.file_type}"
|
|
249
|
-
)
|
|
250
|
-
)
|
|
251
|
-
|
|
252
|
-
# Save results
|
|
253
|
-
result_json = os.path.join(self.result_folder, sample, 'result.json')
|
|
254
|
-
json.dump(results, open(result_json, 'w'), indent=2, allow_nan=False, sort_keys=True)
|
|
255
|
-
|
|
256
|
-
if save_files:
|
|
257
|
-
# Cleanup old extracted and supplementary
|
|
258
|
-
extracted_dir = os.path.join(self.result_folder, sample, 'extracted')
|
|
259
|
-
supplementary_dir = os.path.join(self.result_folder, sample, 'supplementary')
|
|
260
|
-
if os.path.exists(extracted_dir):
|
|
261
|
-
shutil.rmtree(extracted_dir)
|
|
262
|
-
if os.path.exists(supplementary_dir):
|
|
263
|
-
shutil.rmtree(supplementary_dir)
|
|
264
|
-
|
|
265
|
-
# Save extracted files
|
|
266
|
-
for ext in task.extracted:
|
|
267
|
-
target_file = os.path.join(self.result_folder, sample, 'extracted', ext['name'])
|
|
268
|
-
os.makedirs(os.path.dirname(target_file), exist_ok=True)
|
|
269
|
-
shutil.move(ext['path'], target_file)
|
|
270
|
-
|
|
271
|
-
# Save supplementary files
|
|
272
|
-
for ext in task.supplementary:
|
|
273
|
-
target_file = os.path.join(self.result_folder, sample, 'supplementary', ext['name'])
|
|
274
|
-
os.makedirs(os.path.dirname(target_file), exist_ok=True)
|
|
275
|
-
shutil.move(ext['path'], target_file)
|
|
276
|
-
|
|
277
|
-
return results
|
|
278
|
-
finally:
|
|
279
|
-
# Cleanup files
|
|
280
|
-
if cls:
|
|
281
|
-
if os.path.exists(cls.working_directory):
|
|
282
|
-
shutil.rmtree(cls.working_directory)
|
|
283
|
-
cls._cleanup()
|
|
284
|
-
if os.path.exists(file_path):
|
|
285
|
-
os.remove(file_path)
|
|
286
|
-
|
|
287
|
-
def result_list(self):
|
|
288
|
-
return [f for f in os.listdir(self.result_folder)
|
|
289
|
-
if len(f.split("_")[0]) == 64 and os.path.isdir(os.path.join(self.result_folder, f))]
|
|
290
|
-
|
|
291
|
-
def compare_sample_results(self, sample, test_extra=False):
|
|
292
|
-
ih = IssueHelper()
|
|
293
|
-
original_results_file = os.path.join(self.result_folder, sample, 'result.json')
|
|
294
|
-
|
|
295
|
-
if os.path.exists(original_results_file):
|
|
296
|
-
original_results = json.load(open(original_results_file))
|
|
297
|
-
results = self._execute_sample(sample)
|
|
298
|
-
|
|
299
|
-
# Compile the list of issues between the two results
|
|
300
|
-
# Test extra results
|
|
301
|
-
if test_extra and original_results.get('extra', None) != results.get('extra', None):
|
|
302
|
-
ih.add_issue(ih.TYPE_EXTRA, ih.ACTION_CHANGED, "Extra results have changed.")
|
|
303
|
-
|
|
304
|
-
# Extracted files
|
|
305
|
-
self._file_compare(
|
|
306
|
-
ih, ih.TYPE_EXTRACTED, original_results['files']['extracted'],
|
|
307
|
-
results['files']['extracted'])
|
|
308
|
-
# Supplementary files
|
|
309
|
-
self._file_compare(ih, ih.TYPE_SUPPLEMENTARY, original_results['files']['supplementary'],
|
|
310
|
-
results['files']['supplementary'])
|
|
311
|
-
# Heuristics triggered
|
|
312
|
-
self._heuristic_compare(ih, original_results['results']['heuristics'], results['results']['heuristics'])
|
|
313
|
-
# Tags generated
|
|
314
|
-
self._tag_compare(ih, original_results['results']['tags'], results['results']['tags'])
|
|
315
|
-
# Temp submission data generated
|
|
316
|
-
self._temp_data_compare(
|
|
317
|
-
ih, original_results['results']['temp_submission_data'],
|
|
318
|
-
results['results']['temp_submission_data'])
|
|
319
|
-
else:
|
|
320
|
-
ih.append(f"Original result file missing for sample: {sample}")
|
|
321
|
-
|
|
322
|
-
return ih
|
|
323
|
-
|
|
324
|
-
def run_test_comparison(self, sample, test_extra=False):
|
|
325
|
-
# WARNING: This function is only to be run into a pytest context!
|
|
326
|
-
ih = self.compare_sample_results(sample, test_extra=test_extra)
|
|
327
|
-
if ih.has_issues():
|
|
328
|
-
issues = ih.get_issue_list()
|
|
329
|
-
issues.insert(0, "")
|
|
330
|
-
pytest.fail("\n".join(issues))
|
|
331
|
-
|
|
332
|
-
@ staticmethod
|
|
333
|
-
def _heuristic_compare(ih: IssueHelper, original, new):
|
|
334
|
-
oh_map = {x['heur_id']: x for x in original}
|
|
335
|
-
nh_map = {x['heur_id']: x for x in new}
|
|
336
|
-
for heur_id, heur in oh_map.items():
|
|
337
|
-
if heur_id not in nh_map:
|
|
338
|
-
ih.add_issue(ih.TYPE_HEUR, ih.ACTION_MISSING, f"Heuristic #{heur_id} missing from results.")
|
|
339
|
-
else:
|
|
340
|
-
new_heur = nh_map[heur_id]
|
|
341
|
-
for attack_id in heur['attack_ids']:
|
|
342
|
-
if attack_id not in new_heur['attack_ids']:
|
|
343
|
-
ih.add_issue(ih.TYPE_HEUR, ih.ACTION_MISSING,
|
|
344
|
-
f"Attack ID '{attack_id}' missing from heuristic #{heur_id}.")
|
|
345
|
-
for signature in heur['signatures']:
|
|
346
|
-
if signature not in new_heur['signatures']:
|
|
347
|
-
ih.add_issue(ih.TYPE_HEUR, ih.ACTION_MISSING,
|
|
348
|
-
f"Signature '{signature}' missing from heuristic #{heur_id}.")
|
|
349
|
-
|
|
350
|
-
for attack_id in new_heur['attack_ids']:
|
|
351
|
-
if attack_id not in heur['attack_ids']:
|
|
352
|
-
ih.add_issue(ih.TYPE_HEUR, ih.ACTION_ADDED,
|
|
353
|
-
f"Attack ID '{attack_id}' added to heuristic #{heur_id}.")
|
|
354
|
-
for signature in new_heur['signatures']:
|
|
355
|
-
if signature not in heur['signatures']:
|
|
356
|
-
ih.add_issue(ih.TYPE_HEUR, ih.ACTION_ADDED,
|
|
357
|
-
f"Signature '{signature}' added to heuristic #{heur_id}.")
|
|
358
|
-
|
|
359
|
-
for heur_id in nh_map.keys():
|
|
360
|
-
if heur_id not in oh_map:
|
|
361
|
-
ih.add_issue(ih.TYPE_HEUR, ih.ACTION_ADDED, f"Heuristic #{heur_id} added to results.")
|
|
362
|
-
|
|
363
|
-
@ staticmethod
|
|
364
|
-
def _tag_compare(ih: IssueHelper, original, new):
|
|
365
|
-
for tag_type, tags in original.items():
|
|
366
|
-
if tag_type not in new:
|
|
367
|
-
for t in tags:
|
|
368
|
-
ih.add_issue(ih.TYPE_TAGS, ih.ACTION_MISSING,
|
|
369
|
-
f"Tag '{t['value']} [{tag_type}]' missing from the results.")
|
|
370
|
-
else:
|
|
371
|
-
otm = {x['value']: x for x in tags}
|
|
372
|
-
ntm = {x['value']: x for x in new[tag_type]}
|
|
373
|
-
for v, tag in otm.items():
|
|
374
|
-
if v not in ntm:
|
|
375
|
-
ih.add_issue(ih.TYPE_TAGS, ih.ACTION_MISSING,
|
|
376
|
-
f"Tag '{v} [{tag_type}]' missing from the results.")
|
|
377
|
-
else:
|
|
378
|
-
new_tag = ntm[v]
|
|
379
|
-
if tag['heur_id'] != new_tag['heur_id']:
|
|
380
|
-
ih.add_issue(
|
|
381
|
-
ih.TYPE_TAGS, ih.ACTION_CHANGED,
|
|
382
|
-
(
|
|
383
|
-
f"Heuristic ID for tag '{v} [{tag_type}]' has changed "
|
|
384
|
-
f"from {tag['heur_id']} to {new_tag['heur_id']}."
|
|
385
|
-
)
|
|
386
|
-
)
|
|
387
|
-
if tag['signatures'] != new_tag['signatures']:
|
|
388
|
-
ih.add_issue(
|
|
389
|
-
ih.TYPE_TAGS, ih.ACTION_CHANGED,
|
|
390
|
-
(
|
|
391
|
-
f"Associated signatures for tag '{v} [{tag_type}]' have changed "
|
|
392
|
-
f"from {tag['signatures']} to {new_tag['signatures']}."
|
|
393
|
-
)
|
|
394
|
-
)
|
|
395
|
-
|
|
396
|
-
for v in ntm.keys():
|
|
397
|
-
if v not in otm:
|
|
398
|
-
ih.add_issue(ih.TYPE_TAGS, ih.ACTION_ADDED, f"Tag '{v} [{tag_type}]' added to results.")
|
|
399
|
-
|
|
400
|
-
for tag_type, tags in new.items():
|
|
401
|
-
if tag_type not in original:
|
|
402
|
-
for t in tags:
|
|
403
|
-
ih.add_issue(ih.TYPE_TAGS, ih.ACTION_ADDED,
|
|
404
|
-
f"Tag '{t['value']} [{tag_type}]' added to the results.")
|
|
405
|
-
|
|
406
|
-
@ staticmethod
|
|
407
|
-
def _temp_data_compare(ih: IssueHelper, original, new):
|
|
408
|
-
for k, v in original.items():
|
|
409
|
-
if k not in new:
|
|
410
|
-
ih.add_issue(ih.TYPE_TEMP, ih.ACTION_MISSING,
|
|
411
|
-
f"Temporary submission data with key '{k}' is missing from the results.")
|
|
412
|
-
elif v != new[k]:
|
|
413
|
-
ih.add_issue(ih.TYPE_TEMP, ih.ACTION_CHANGED,
|
|
414
|
-
f"Value of temporary submission data with key '{k}' has changed.")
|
|
415
|
-
|
|
416
|
-
for k, v in new.items():
|
|
417
|
-
if k not in original:
|
|
418
|
-
ih.add_issue(ih.TYPE_TEMP, ih.ACTION_ADDED,
|
|
419
|
-
f"Temporary submission data with key '{k}' was added to the results.")
|
|
420
|
-
|
|
421
|
-
@ staticmethod
|
|
422
|
-
def _file_compare(ih: IssueHelper, f_type, original, new):
|
|
423
|
-
oh_map = {x['sha256']: x['name'] for x in original}
|
|
424
|
-
on_map = {x['name']: x['sha256'] for x in original}
|
|
425
|
-
nh_map = {x['sha256']: x['name'] for x in new}
|
|
426
|
-
nn_map = {x['name']: x['sha256'] for x in new}
|
|
427
|
-
|
|
428
|
-
for sha256, name in oh_map.items():
|
|
429
|
-
if sha256 not in nh_map:
|
|
430
|
-
if name not in nn_map:
|
|
431
|
-
ih.add_issue(f_type, ih.ACTION_MISSING, f"File '{name} [{sha256}]' missing from the file list.")
|
|
432
|
-
continue
|
|
433
|
-
|
|
434
|
-
if sha256 != nn_map[name]:
|
|
435
|
-
ih.add_issue(
|
|
436
|
-
f_type,
|
|
437
|
-
ih.ACTION_CHANGED,
|
|
438
|
-
f"The sha256 of the file '{name}' has changed. {sha256} -> {nn_map[name]}"
|
|
439
|
-
)
|
|
440
|
-
continue
|
|
441
|
-
|
|
442
|
-
if name != nh_map[sha256]:
|
|
443
|
-
ih.add_issue(
|
|
444
|
-
f_type,
|
|
445
|
-
ih.ACTION_CHANGED,
|
|
446
|
-
f"The name of the file '{sha256}' has changed. {name} -> {nh_map[sha256]}"
|
|
447
|
-
)
|
|
448
|
-
continue
|
|
449
|
-
|
|
450
|
-
for sha256, name in nh_map.items():
|
|
451
|
-
if sha256 not in oh_map and name not in on_map:
|
|
452
|
-
ih.add_issue(f_type, ih.ACTION_ADDED, f"File '{name} [{sha256}]' added to the file list.")
|
|
453
|
-
|
|
454
|
-
def regenerate_results(self, save_files=False, sample_sha256=""):
|
|
455
|
-
for f in self.result_list():
|
|
456
|
-
if sample_sha256 and f != sample_sha256:
|
|
457
|
-
print(f"{sample_sha256} requested. Skipping {f}...")
|
|
458
|
-
continue
|
|
459
|
-
try:
|
|
460
|
-
print(f"Executing {f}")
|
|
461
|
-
self._execute_sample(f, save=True, save_files=save_files)
|
|
462
|
-
except FileMissing:
|
|
463
|
-
print(f"[W] File {f} was not found in any of the following locations: {', '.join(self.locations)}")
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
|
|
3
|
-
from assemblyline.common.importing import load_module_by_path
|
|
4
|
-
from assemblyline_v4_service.testing.helper import TestHelper
|
|
5
|
-
|
|
6
|
-
required_env = [
|
|
7
|
-
'SERVICE_MANIFEST_PATH',
|
|
8
|
-
'SERVICE_TESTING_RESULT_FOLDER',
|
|
9
|
-
'SERVICE_PATH'
|
|
10
|
-
]
|
|
11
|
-
|
|
12
|
-
optional_env = [
|
|
13
|
-
'SERVICE_TESTING_EXTRA_SAMPLE_FOLDER',
|
|
14
|
-
'FULL_SAMPLES_LOCATION',
|
|
15
|
-
'SERVICE_TESTING_SAVE_FILES'
|
|
16
|
-
]
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def run():
|
|
20
|
-
for env in required_env:
|
|
21
|
-
if os.environ.get(env, None) is None:
|
|
22
|
-
print(f"[E] You must set {env} environement variable.")
|
|
23
|
-
exit(1)
|
|
24
|
-
|
|
25
|
-
for env in optional_env:
|
|
26
|
-
if os.environ.get(env, None) is None:
|
|
27
|
-
print(f"[W] {env} environement variable is not set, it should probably be...")
|
|
28
|
-
|
|
29
|
-
th = TestHelper(load_module_by_path(os.environ['SERVICE_PATH']),
|
|
30
|
-
os.environ['SERVICE_TESTING_RESULT_FOLDER'],
|
|
31
|
-
os.environ.get('SERVICE_TESTING_EXTRA_SAMPLE_FOLDER', None))
|
|
32
|
-
save_files = os.environ.get('SERVICE_TESTING_SAVE_FILES', 'false').lower() == 'true'
|
|
33
|
-
th.regenerate_results(save_files=save_files)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if __name__ == "__main__":
|
|
37
|
-
run()
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
assemblyline_v4_service/VERSION,sha256=v2o1_8Y2gl2Ul8UPg_qSFCAsGQPVybam41koH3Oao-c,9
|
|
2
|
-
assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
|
|
4
|
-
assemblyline_v4_service/run_privileged_service.py,sha256=9uTfHetXR5G-EDKMDrgfWUOw34yr64-cj6Cm9eZaCbQ,14547
|
|
5
|
-
assemblyline_v4_service/run_service.py,sha256=RCqxdm-OAwJhl15BnKFkuavpQ5k6eTX3ZGeSna5JJBw,5557
|
|
6
|
-
assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
assemblyline_v4_service/common/api.py,sha256=U908p3wlW9fEydx77GgI2E-6wW6T8Nc3R91nNOKU0H0,4453
|
|
8
|
-
assemblyline_v4_service/common/base.py,sha256=qKhNpcD-FwV0R-5EWDWWXgr2Y-2dciVY6IIQsprenQ0,13187
|
|
9
|
-
assemblyline_v4_service/common/dynamic_service_helper.py,sha256=OhxSnSOjUB_iSkMahHbPQEll9sl5wLuCFwwykvV-Kno,147461
|
|
10
|
-
assemblyline_v4_service/common/helper.py,sha256=Fgimk8DhnS23aijTGewA1HwvPoAM61UUbHlrGBnSzL0,3290
|
|
11
|
-
assemblyline_v4_service/common/icap.py,sha256=phT3CT5uII3Qm90Nzi4O-eDkQ2jmr3zHcVVra4sqYSc,5376
|
|
12
|
-
assemblyline_v4_service/common/keytool_parse.py,sha256=e829hrNNG5LFw1kjLsYVZsafCm2S3NpgM6jBc6JKawY,2219
|
|
13
|
-
assemblyline_v4_service/common/ontology_helper.py,sha256=uiwc5cfPDAesEDYKk7etzCMTGQNVwhNrO3mWLdB2520,7793
|
|
14
|
-
assemblyline_v4_service/common/request.py,sha256=p8A9boDZ6KuVxl3EdhvaU1D_5K6_gAVoIbJYDz8TzjA,9711
|
|
15
|
-
assemblyline_v4_service/common/result.py,sha256=9OGfWTCnBtow31Ft03us9Ew_2pXyDydUTP4iio9Qg8Q,30349
|
|
16
|
-
assemblyline_v4_service/common/safelist_helper.py,sha256=QHTuG8q52o3U307AADPgrIgug7aYFK2uQE4-EtWG3yQ,3037
|
|
17
|
-
assemblyline_v4_service/common/section_reducer.py,sha256=JJOT7eFfBn4hFJKHY9UeVEbHS-E8FpmQ_dPZC-dWla0,1513
|
|
18
|
-
assemblyline_v4_service/common/tag_helper.py,sha256=om3TVPY_XDeFDqVW2iUA349xbljSAy5tv667jCiA7JI,4186
|
|
19
|
-
assemblyline_v4_service/common/tag_reducer.py,sha256=T6_l6T7EQKX1pYvTZ7swO90SifUPwwEfPWNUO-IZjjk,11102
|
|
20
|
-
assemblyline_v4_service/common/task.py,sha256=PiYJdL2Qd2dKzwJ_FzWv6PBMvq-gHiVZq05ZQCjtREA,12480
|
|
21
|
-
assemblyline_v4_service/common/utils.py,sha256=K0UNJ3cugDHoqAgt6zvJ3JYEAiyT1XM2efd7XTNRPK8,2338
|
|
22
|
-
assemblyline_v4_service/common/balbuzard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
-
assemblyline_v4_service/common/balbuzard/balbuzard.py,sha256=JnIs3vkfp6uQGtxGL9mklM0TG4kP3jaRw2RE5pgrBzo,25161
|
|
24
|
-
assemblyline_v4_service/common/balbuzard/bbcrack.py,sha256=6v2xe_VI5C3E9Ga8rD-FAxMKok7T-UZkjuxYfo9KQHw,29180
|
|
25
|
-
assemblyline_v4_service/common/balbuzard/patterns.py,sha256=NYi38A_EADKA3fRFkx1u8K5WqPoW709d2LvomHJerkg,34120
|
|
26
|
-
assemblyline_v4_service/common/extractor/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
27
|
-
assemblyline_v4_service/common/extractor/base64.py,sha256=mgSWJr_xKRnLO1NwoJf27HAO5LnFkOb8wu5ZyNp9WVU,3091
|
|
28
|
-
assemblyline_v4_service/common/extractor/ocr.py,sha256=Pm47dRlMu66kwuPd6WQO5UPziMusJt8pPSq8Sj4YxiA,4385
|
|
29
|
-
assemblyline_v4_service/common/extractor/pe_file.py,sha256=d2ASmgibA8mSNcfVz2glgABxqyswnpGgo0nNNwSfREg,1414
|
|
30
|
-
assemblyline_v4_service/common/pestudio/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
|
-
assemblyline_v4_service/common/pestudio/xml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
-
assemblyline_v4_service/common/pestudio/xml/features.xml,sha256=q883UJxIq20mk88pxYnv2nKeG5dFiq-mhYH7D45i9lE,196476
|
|
33
|
-
assemblyline_v4_service/common/pestudio/xml/functions.xml,sha256=QnbWwvvKeugnIU4XFYh8jeuP-eGl_h4z5KKihvpnCNU,350312
|
|
34
|
-
assemblyline_v4_service/common/pestudio/xml/languages.xml,sha256=NLWwcGIVTWOQh7vvzP5yYPAstJJMsRXzNAv54Kly42M,20008
|
|
35
|
-
assemblyline_v4_service/common/pestudio/xml/resources.xml,sha256=bYP-AVbq_Dlhgj_gNC2o0LSqc0AdlJHxJ8-G8Ih0MIc,40129
|
|
36
|
-
assemblyline_v4_service/common/pestudio/xml/signatures.xml,sha256=r5FLFgmWejRdqPdoPQCFTt31Tllc64o81geaN1SbwS4,1255658
|
|
37
|
-
assemblyline_v4_service/common/pestudio/xml/strings.xml,sha256=kRU8WbCcU1RckM6oCFeUVMdpOxZjJDDTMIIor8k2ru0,102459
|
|
38
|
-
assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
|
-
assemblyline_v4_service/dev/run_service_once.py,sha256=4gnb09WeKXlyWQKCQdH4SoL4xtfIRWq_9nyIiECrJ7g,10592
|
|
40
|
-
assemblyline_v4_service/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
-
assemblyline_v4_service/testing/helper.py,sha256=f0-qBtgR0vWZBpEV9sPfcworLtdh4h_CcoAofHlOtZE,19711
|
|
42
|
-
assemblyline_v4_service/testing/regenerate_results.py,sha256=Cbp2CMAxbF3kz5vxEPPCxrgUp1Vl3Tz6e46aUhg_I4U,1101
|
|
43
|
-
assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
44
|
-
assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
|
|
45
|
-
assemblyline_v4_service/updater/app.py,sha256=OF-G7J8IYwTjdGw_BI9numBicZBN5lspm8Eqb3qbIl8,3229
|
|
46
|
-
assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
|
|
47
|
-
assemblyline_v4_service/updater/helper.py,sha256=JD0gX3KHY-wvsFjTbWkT83F0d5Up3OfubMPinuNzbTQ,9069
|
|
48
|
-
assemblyline_v4_service/updater/updater.py,sha256=gTFNN8Xow1HnlUcftWG30-ULXK5_FfJynZBnf7i4aIQ,29528
|
|
49
|
-
assemblyline_v4_service-4.4.0.24.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
|
|
50
|
-
assemblyline_v4_service-4.4.0.24.dist-info/METADATA,sha256=hzqTUPFYhVc4sa5TpbyoSZ1GrzzkNrzmrVn4swKDdwY,9328
|
|
51
|
-
assemblyline_v4_service-4.4.0.24.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
52
|
-
assemblyline_v4_service-4.4.0.24.dist-info/top_level.txt,sha256=Ut5IqePObcxlJ8rv2--dOAzYbxzqlllfiV_51cbqjbA,24
|
|
53
|
-
assemblyline_v4_service-4.4.0.24.dist-info/RECORD,,
|
{assemblyline_v4_service-4.4.0.24.dist-info → assemblyline_v4_service-4.4.0.26.dist-info}/LICENCE.md
RENAMED
|
File without changes
|
{assemblyline_v4_service-4.4.0.24.dist-info → assemblyline_v4_service-4.4.0.26.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|