assemblyline-v4-service 4.5.0.dev5__py3-none-any.whl → 4.5.0.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of assemblyline-v4-service might be problematic. Click here for more details.
- assemblyline_v4_service/VERSION +1 -1
- assemblyline_v4_service/updater/client.py +21 -26
- assemblyline_v4_service/updater/updater.py +127 -240
- {assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/METADATA +1 -5
- {assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/RECORD +8 -8
- {assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/LICENCE.md +0 -0
- {assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/WHEEL +0 -0
- {assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/top_level.txt +0 -0
assemblyline_v4_service/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.5.0.
|
|
1
|
+
4.5.0.dev7
|
|
@@ -1,24 +1,22 @@
|
|
|
1
1
|
import os
|
|
2
2
|
|
|
3
|
-
from assemblyline.common import forge
|
|
4
3
|
from assemblyline.odm.models.signature import Signature as SignatureModel
|
|
5
|
-
from
|
|
6
|
-
from
|
|
4
|
+
from assemblyline_core.badlist_client import BadlistClient
|
|
5
|
+
from assemblyline_core.safelist_client import SafelistClient
|
|
6
|
+
from assemblyline_core.signature_client import SignatureClient
|
|
7
7
|
|
|
8
8
|
from typing import Any, Dict, List, Union
|
|
9
9
|
|
|
10
10
|
SIGNATURE_UPDATE_BATCH = int(os.environ.get('SIGNATURE_UPDATE_BATCH', '1000'))
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class
|
|
14
|
-
def __init__(self,
|
|
15
|
-
super().__init__(
|
|
16
|
-
self.datastore = datastore
|
|
17
|
-
if not datastore:
|
|
18
|
-
self.datastore = forge.get_datastore()
|
|
13
|
+
class SyncableSignature(SignatureClient):
|
|
14
|
+
def __init__(self, datastore, config=None):
|
|
15
|
+
super().__init__(datastore, config)
|
|
19
16
|
self.sync = False
|
|
20
17
|
|
|
21
|
-
def add_update_many(self, source: str, sig_type: str, data: List[Union[dict, SignatureModel]],
|
|
18
|
+
def add_update_many(self, source: str, sig_type: str, data: List[Union[dict, SignatureModel]],
|
|
19
|
+
dedup_name: bool = True) -> Dict[str, Any]:
|
|
22
20
|
# This version of the API allows to sync signatures with the system by making direct changes to the datastore
|
|
23
21
|
# Signatures that no longer exist at the source will be DISABLED to maintain active synchronicity,
|
|
24
22
|
# but users can always re-deploy signatures if desired
|
|
@@ -51,18 +49,19 @@ class Signature(SignatureAPI):
|
|
|
51
49
|
# Get the list of signatures that currently existing in the system for the source
|
|
52
50
|
existing_signature_ids = set([
|
|
53
51
|
i.id for i in self.datastore.signature.stream_search(f"source:{source} AND type:{sig_type}", fl='id')
|
|
54
|
-
|
|
52
|
+
])
|
|
55
53
|
|
|
56
54
|
# Find the signature IDs that don't exist at this source anymore and disable them
|
|
57
55
|
for missing_signature_id in (existing_signature_ids - current_signature_ids):
|
|
58
56
|
missing_signature = self.datastore.signature.get(missing_signature_id)
|
|
59
|
-
if missing_signature.state_change_user in ['update_service_account', None] and
|
|
57
|
+
if missing_signature.state_change_user in ['update_service_account', None] and \
|
|
58
|
+
missing_signature.status != 'DISABLED':
|
|
60
59
|
# Only disable signature if it doesn't seem to be in use/altered by a (real) user
|
|
61
60
|
self.datastore.signature.update(missing_signature_id,
|
|
62
61
|
[(self.datastore.signature.UPDATE_SET, 'status', 'DISABLED'),
|
|
63
62
|
(self.datastore.signature.UPDATE_SET, 'last_modified', 'NOW')])
|
|
64
63
|
|
|
65
|
-
# Proceed with adding/updating signatures
|
|
64
|
+
# Proceed with adding/updating signatures
|
|
66
65
|
if len(data) < SIGNATURE_UPDATE_BATCH:
|
|
67
66
|
# Update all of them in a single batch
|
|
68
67
|
return super().add_update_many(source, sig_type, data, dedup_name)
|
|
@@ -81,23 +80,19 @@ class Signature(SignatureAPI):
|
|
|
81
80
|
|
|
82
81
|
# Split up data into batches to avoid server timeouts handling requests
|
|
83
82
|
batch_num = 0
|
|
84
|
-
start = batch_num*SIGNATURE_UPDATE_BATCH
|
|
83
|
+
start = batch_num * SIGNATURE_UPDATE_BATCH
|
|
85
84
|
while start < len(data):
|
|
86
|
-
end = (batch_num+1)*SIGNATURE_UPDATE_BATCH
|
|
85
|
+
end = (batch_num + 1) * SIGNATURE_UPDATE_BATCH
|
|
87
86
|
update_response(super().add_update_many(source, sig_type, data[start:end], dedup_name))
|
|
88
87
|
batch_num += 1
|
|
89
|
-
start = batch_num*SIGNATURE_UPDATE_BATCH
|
|
88
|
+
start = batch_num * SIGNATURE_UPDATE_BATCH
|
|
90
89
|
|
|
91
90
|
return response
|
|
92
91
|
|
|
93
92
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
apikey, verify, timeout, oauth)
|
|
101
|
-
# Override Signature module with custom implementation
|
|
102
|
-
client.signature = Signature(client._connection, datastore=datastore)
|
|
103
|
-
return client
|
|
93
|
+
class UpdaterClient(object):
|
|
94
|
+
def __init__(self, datastore) -> None:
|
|
95
|
+
self.datastore = datastore
|
|
96
|
+
self.badlist = BadlistClient(datastore)
|
|
97
|
+
self.safelist = SafelistClient(datastore)
|
|
98
|
+
self.signature = SyncableSignature(datastore)
|
|
@@ -7,34 +7,24 @@ import logging
|
|
|
7
7
|
import time
|
|
8
8
|
import json
|
|
9
9
|
import tempfile
|
|
10
|
-
import string
|
|
11
|
-
import random
|
|
12
10
|
import tarfile
|
|
13
11
|
import threading
|
|
14
12
|
import subprocess
|
|
15
13
|
import hashlib
|
|
16
|
-
from contextlib import contextmanager
|
|
17
14
|
from queue import Queue
|
|
18
|
-
from zipfile import ZipFile
|
|
15
|
+
from zipfile import ZipFile
|
|
19
16
|
|
|
20
17
|
from assemblyline.common import forge, log as al_log
|
|
21
18
|
from assemblyline.common.isotime import epoch_to_iso, now_as_iso
|
|
22
|
-
from assemblyline.common.identify import zip_ident
|
|
23
|
-
from assemblyline.common.security import get_password_hash
|
|
24
19
|
from assemblyline.odm.messages.changes import Operation, ServiceChange, SignatureChange
|
|
25
20
|
from assemblyline.remote.datatypes.events import EventSender, EventWatcher
|
|
26
21
|
|
|
27
22
|
from assemblyline_core.server_base import ThreadedCoreBase, ServiceStage
|
|
28
23
|
from assemblyline.odm.models.service import Service, UpdateSource
|
|
29
24
|
from assemblyline.remote.datatypes.hash import Hash
|
|
30
|
-
from assemblyline.common.security import get_random_password, get_password_hash
|
|
31
|
-
from assemblyline.remote.datatypes.lock import Lock
|
|
32
|
-
from assemblyline.odm.models.user import User
|
|
33
|
-
from assemblyline.odm.models.user_settings import UserSettings
|
|
34
25
|
|
|
35
|
-
from assemblyline_client import Client4
|
|
36
26
|
from assemblyline_v4_service.common.base import SIGNATURES_META_FILENAME
|
|
37
|
-
from assemblyline_v4_service.updater.client import
|
|
27
|
+
from assemblyline_v4_service.updater.client import UpdaterClient
|
|
38
28
|
from assemblyline_v4_service.updater.helper import url_download, git_clone_repo, SkipSource, filter_downloads
|
|
39
29
|
|
|
40
30
|
if typing.TYPE_CHECKING:
|
|
@@ -53,8 +43,6 @@ SOURCE_EXTRA_KEY = 'source_extra'
|
|
|
53
43
|
SOURCE_STATUS_KEY = 'status'
|
|
54
44
|
SOURCE_UPDATE_ATTEMPT_DELAY_BASE = int(os.environ.get("SOURCE_UPDATE_ATTEMPT_DELAY_BASE", "5"))
|
|
55
45
|
SOURCE_UPDATE_ATTEMPT_MAX_RETRY = int(os.environ.get("SOURCE_UPDATE_ATTEMPT_MAX_RETRY", "3"))
|
|
56
|
-
UI_SERVER = os.getenv('UI_SERVER', 'https://nginx')
|
|
57
|
-
UI_SERVER_ROOT_CA = os.environ.get('UI_SERVER_ROOT_CA', '/etc/assemblyline/ssl/al_root-ca.crt')
|
|
58
46
|
UPDATER_DIR = os.getenv('UPDATER_DIR', os.path.join(tempfile.gettempdir(), 'updater'))
|
|
59
47
|
UPDATER_API_ROLES = ['badlist_manage', 'signature_import', 'signature_download',
|
|
60
48
|
'signature_view', 'safelist_manage', 'apikey_access', 'signature_manage']
|
|
@@ -63,29 +51,6 @@ STATUS_FILE = '/tmp/status'
|
|
|
63
51
|
classification = forge.get_classification()
|
|
64
52
|
|
|
65
53
|
|
|
66
|
-
@contextmanager
|
|
67
|
-
def temporary_api_key(ds: AssemblylineDatastore, user_name: str, permissions=('R', 'W')):
|
|
68
|
-
"""Creates a context where a temporary API key is available."""
|
|
69
|
-
with Lock(f'user-{user_name}', timeout=10):
|
|
70
|
-
name = ''.join(random.choices(string.ascii_lowercase, k=20))
|
|
71
|
-
random_pass = get_random_password(length=48)
|
|
72
|
-
user = ds.user.get(user_name)
|
|
73
|
-
user.apikeys[name] = {
|
|
74
|
-
"password": get_password_hash(random_pass),
|
|
75
|
-
"acl": permissions,
|
|
76
|
-
"roles": UPDATER_API_ROLES
|
|
77
|
-
}
|
|
78
|
-
ds.user.save(user_name, user)
|
|
79
|
-
|
|
80
|
-
try:
|
|
81
|
-
yield f"{name}:{random_pass}"
|
|
82
|
-
finally:
|
|
83
|
-
with Lock(f'user-{user_name}', timeout=10):
|
|
84
|
-
user = ds.user.get(user_name)
|
|
85
|
-
user.apikeys.pop(name)
|
|
86
|
-
ds.user.save(user_name, user)
|
|
87
|
-
|
|
88
|
-
|
|
89
54
|
# A Queue derivative that respects uniqueness of items as well as order
|
|
90
55
|
class UniqueQueue(Queue):
|
|
91
56
|
# Put a new item in the queue
|
|
@@ -127,6 +92,7 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
127
92
|
self.event_sender = EventSender('changes.services',
|
|
128
93
|
host=self.config.core.redis.nonpersistent.host,
|
|
129
94
|
port=self.config.core.redis.nonpersistent.port)
|
|
95
|
+
self.client = UpdaterClient(self.datastore)
|
|
130
96
|
|
|
131
97
|
self.service_change_watcher = EventWatcher(self.redis, deserializer=ServiceChange.deserialize)
|
|
132
98
|
self.service_change_watcher.register(f'changes.services.{SERVICE_NAME}', self._handle_service_change_event)
|
|
@@ -164,9 +130,6 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
164
130
|
status_query = ' OR '.join([f'status:{s}' for s in self.statuses])
|
|
165
131
|
self.signatures_query = f"type:{self.updater_type} AND ({status_query})"
|
|
166
132
|
|
|
167
|
-
# SSL configuration to UI_SERVER
|
|
168
|
-
self.verify = None if not os.path.exists(UI_SERVER_ROOT_CA) else UI_SERVER_ROOT_CA
|
|
169
|
-
|
|
170
133
|
def trigger_update(self):
|
|
171
134
|
self.source_update_flag.set()
|
|
172
135
|
|
|
@@ -349,177 +312,131 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
349
312
|
if not os.path.exists(UPDATER_DIR):
|
|
350
313
|
os.makedirs(UPDATER_DIR)
|
|
351
314
|
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
self.
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
self.
|
|
372
|
-
|
|
373
|
-
self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}")
|
|
374
|
-
|
|
375
|
-
extracted_zip = False
|
|
376
|
-
attempt = 0
|
|
377
|
-
|
|
378
|
-
# Sometimes a zip file isn't always returned, will affect
|
|
379
|
-
# service's use of signature source. Patience..
|
|
380
|
-
while not extracted_zip and attempt < 5:
|
|
381
|
-
temp_zip_file = os.path.join(output_directory, 'temp.zip')
|
|
382
|
-
al_client.signature.download(output=temp_zip_file, query=self.signatures_query)
|
|
383
|
-
self.log.debug(f"Downloading update to {temp_zip_file}")
|
|
384
|
-
if os.path.exists(temp_zip_file) and os.path.getsize(temp_zip_file) > 0:
|
|
385
|
-
self.log.debug(
|
|
386
|
-
f"File type ({os.path.getsize(temp_zip_file)}B): {zip_ident(temp_zip_file, 'unknown')}")
|
|
387
|
-
try:
|
|
388
|
-
with ZipFile(temp_zip_file, 'r') as zip_f:
|
|
389
|
-
zip_f.extractall(output_directory)
|
|
390
|
-
extracted_zip = True
|
|
391
|
-
self.log.info("Zip extracted.")
|
|
392
|
-
except BadZipFile:
|
|
393
|
-
attempt += 1
|
|
394
|
-
self.log.warning(f"[{attempt}/5] Bad zip. Trying again after 30s...")
|
|
395
|
-
time.sleep(30)
|
|
396
|
-
except Exception as e:
|
|
397
|
-
self.log.error(f'Problem while extracting signatures to disk: {e}')
|
|
398
|
-
break
|
|
399
|
-
|
|
400
|
-
os.remove(temp_zip_file)
|
|
401
|
-
|
|
402
|
-
if extracted_zip:
|
|
403
|
-
self.log.info("New ruleset successfully downloaded and ready to use")
|
|
404
|
-
self.serve_directory(output_directory, time_keeper, al_client)
|
|
405
|
-
else:
|
|
406
|
-
self.log.error("Signatures aren't saved to disk.")
|
|
407
|
-
shutil.rmtree(output_directory, ignore_errors=True)
|
|
408
|
-
if os.path.exists(time_keeper):
|
|
409
|
-
os.unlink(time_keeper)
|
|
410
|
-
else:
|
|
411
|
-
self.log.info("No signature updates available.")
|
|
412
|
-
shutil.rmtree(output_directory, ignore_errors=True)
|
|
413
|
-
if os.path.exists(time_keeper):
|
|
414
|
-
os.unlink(time_keeper)
|
|
315
|
+
# Create a temporary file for the time keeper
|
|
316
|
+
time_keeper = tempfile.NamedTemporaryFile(prefix="time_keeper_", dir=UPDATER_DIR, delete=False)
|
|
317
|
+
time_keeper.close()
|
|
318
|
+
time_keeper = time_keeper.name
|
|
319
|
+
|
|
320
|
+
if self._service.update_config.generates_signatures:
|
|
321
|
+
output_directory = tempfile.mkdtemp(prefix="update_dir_", dir=UPDATER_DIR)
|
|
322
|
+
|
|
323
|
+
# Check if new signatures have been added
|
|
324
|
+
self.log.info("Check for new signatures.")
|
|
325
|
+
if self.client.signature.update_available(since=epoch_to_iso(old_update_time) or None,
|
|
326
|
+
sig_type=self.updater_type):
|
|
327
|
+
self.log.info("An update is available for download from the datastore")
|
|
328
|
+
|
|
329
|
+
self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}")
|
|
330
|
+
|
|
331
|
+
with ZipFile(self.client.signature.download(self.signatures_query), 'r') as zip_f:
|
|
332
|
+
zip_f.extractall(output_directory)
|
|
333
|
+
self.log.info("New ruleset successfully downloaded and ready to use")
|
|
334
|
+
self.serve_directory(output_directory, time_keeper)
|
|
415
335
|
else:
|
|
416
|
-
|
|
417
|
-
|
|
336
|
+
self.log.info("No signature updates available.")
|
|
337
|
+
shutil.rmtree(output_directory, ignore_errors=True)
|
|
338
|
+
if os.path.exists(time_keeper):
|
|
339
|
+
os.unlink(time_keeper)
|
|
340
|
+
else:
|
|
341
|
+
output_directory = self.prepare_output_directory()
|
|
342
|
+
self.serve_directory(output_directory, time_keeper)
|
|
418
343
|
|
|
419
344
|
def do_source_update(self, service: Service) -> None:
|
|
420
|
-
self.log.info(f"Connecting to Assemblyline API: {UI_SERVER}...")
|
|
421
345
|
run_time = time.time()
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
#
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
#
|
|
475
|
-
output =
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
except SkipSource:
|
|
507
|
-
# This source hasn't changed, no need to re-import into Assemblyline
|
|
508
|
-
self.log.info(f'No new {self.updater_type} rule files to process for {source_name}')
|
|
509
|
-
if source_name in previous_hashes:
|
|
510
|
-
files_sha256[source_name] = previous_hashes[source_name]
|
|
511
|
-
seen_fetches[uri] = "skipped"
|
|
512
|
-
self.push_status("DONE", "Skipped.")
|
|
513
|
-
break
|
|
514
|
-
except Exception as e:
|
|
515
|
-
# There was an issue with this source, report and continue to the next
|
|
516
|
-
self.log.error(f"Problem with {source['name']}: {e}")
|
|
517
|
-
self.push_status("ERROR", str(e))
|
|
518
|
-
continue
|
|
519
|
-
|
|
520
|
-
self.set_source_update_time(run_time)
|
|
521
|
-
self.set_source_extra(files_sha256)
|
|
346
|
+
with tempfile.TemporaryDirectory() as update_dir:
|
|
347
|
+
# Parse updater configuration
|
|
348
|
+
previous_hashes: dict[str, dict[str, str]] = self.get_source_extra()
|
|
349
|
+
sources: dict[str, UpdateSource] = {_s['name']: _s for _s in service.update_config.sources}
|
|
350
|
+
files_sha256: dict[str, dict[str, str]] = {}
|
|
351
|
+
|
|
352
|
+
# Map already visited URIs to download paths (avoid re-cloning/re-downloads)
|
|
353
|
+
seen_fetches = dict()
|
|
354
|
+
|
|
355
|
+
# Go through each source queued and download file
|
|
356
|
+
while self.update_queue.qsize():
|
|
357
|
+
update_attempt = -1
|
|
358
|
+
source_name = self.update_queue.get()
|
|
359
|
+
while update_attempt < SOURCE_UPDATE_ATTEMPT_MAX_RETRY:
|
|
360
|
+
# Introduce an exponential delay between each attempt
|
|
361
|
+
time.sleep(SOURCE_UPDATE_ATTEMPT_DELAY_BASE**update_attempt)
|
|
362
|
+
update_attempt += 1
|
|
363
|
+
|
|
364
|
+
# Set current source for pushing state to UI
|
|
365
|
+
self._current_source = source_name
|
|
366
|
+
source_obj = sources[source_name]
|
|
367
|
+
old_update_time = self.get_source_update_time()
|
|
368
|
+
|
|
369
|
+
self.push_status("UPDATING", "Starting..")
|
|
370
|
+
source = source_obj.as_primitives()
|
|
371
|
+
uri: str = source['uri']
|
|
372
|
+
default_classification = source.get('default_classification', classification.UNRESTRICTED)
|
|
373
|
+
# Enable signature syncing if the source specifies it
|
|
374
|
+
self.client.signature.sync = source.get('sync', False)
|
|
375
|
+
|
|
376
|
+
try:
|
|
377
|
+
self.push_status("UPDATING", "Pulling..")
|
|
378
|
+
output = None
|
|
379
|
+
seen_fetch = seen_fetches.get(uri)
|
|
380
|
+
if seen_fetch == 'skipped':
|
|
381
|
+
# Skip source if another source says nothing has changed
|
|
382
|
+
raise SkipSource
|
|
383
|
+
elif seen_fetch and os.path.exists(seen_fetch):
|
|
384
|
+
# We've already fetched something from the same URI, re-use downloaded path
|
|
385
|
+
self.log.info(f'Already visited {uri} in this run. Using cached download path..')
|
|
386
|
+
output = seen_fetches[uri]
|
|
387
|
+
else:
|
|
388
|
+
# Pull sources from external locations (method depends on the URL)
|
|
389
|
+
try:
|
|
390
|
+
# First we'll attempt by performing a Git clone
|
|
391
|
+
# (since not all services hint at being a repository in their URL),
|
|
392
|
+
output = git_clone_repo(source, old_update_time, self.log, update_dir)
|
|
393
|
+
except SkipSource:
|
|
394
|
+
raise
|
|
395
|
+
except Exception as git_ex:
|
|
396
|
+
# Should that fail, we'll attempt a direct-download using Python Requests
|
|
397
|
+
if not uri.endswith('.git'):
|
|
398
|
+
# Proceed with direct download, raise exception as required if necessary
|
|
399
|
+
output = url_download(source, old_update_time, self.log, update_dir)
|
|
400
|
+
else:
|
|
401
|
+
# Raise Git Exception
|
|
402
|
+
raise git_ex
|
|
403
|
+
# Add output path to the list of seen fetches in this run
|
|
404
|
+
seen_fetches[uri] = output
|
|
405
|
+
|
|
406
|
+
files = filter_downloads(output, source['pattern'], self.default_pattern)
|
|
407
|
+
|
|
408
|
+
# Add to collection of sources for caching purposes
|
|
409
|
+
self.log.info(f"Found new {self.updater_type} rule files to process for {source_name}!")
|
|
410
|
+
validated_files = list()
|
|
411
|
+
for file, sha256 in files:
|
|
412
|
+
files_sha256.setdefault(source_name, {})
|
|
413
|
+
if previous_hashes.get(
|
|
414
|
+
source_name, {}).get(
|
|
415
|
+
file, None) != sha256 and self.is_valid(file):
|
|
416
|
+
files_sha256[source_name][file] = sha256
|
|
417
|
+
validated_files.append((file, sha256))
|
|
418
|
+
|
|
419
|
+
self.push_status("UPDATING", "Importing..")
|
|
420
|
+
# Import into Assemblyline
|
|
421
|
+
self.import_update(validated_files, self.client, source_name, default_classification)
|
|
422
|
+
self.push_status("DONE", "Signature(s) Imported.")
|
|
423
|
+
except SkipSource:
|
|
424
|
+
# This source hasn't changed, no need to re-import into Assemblyline
|
|
425
|
+
self.log.info(f'No new {self.updater_type} rule files to process for {source_name}')
|
|
426
|
+
if source_name in previous_hashes:
|
|
427
|
+
files_sha256[source_name] = previous_hashes[source_name]
|
|
428
|
+
seen_fetches[uri] = "skipped"
|
|
429
|
+
self.push_status("DONE", "Skipped.")
|
|
522
430
|
break
|
|
431
|
+
except Exception as e:
|
|
432
|
+
# There was an issue with this source, report and continue to the next
|
|
433
|
+
self.log.error(f"Problem with {source['name']}: {e}")
|
|
434
|
+
self.push_status("ERROR", str(e))
|
|
435
|
+
continue
|
|
436
|
+
|
|
437
|
+
self.set_source_update_time(run_time)
|
|
438
|
+
self.set_source_extra(files_sha256)
|
|
439
|
+
break
|
|
523
440
|
self.set_active_config_hash(self.config_hash(service))
|
|
524
441
|
self.local_update_flag.set()
|
|
525
442
|
|
|
@@ -528,8 +445,7 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
528
445
|
return True
|
|
529
446
|
|
|
530
447
|
# Define how your source update gets imported into Assemblyline
|
|
531
|
-
def import_update(self, files_sha256: List[Tuple[str, str]],
|
|
532
|
-
default_classification=None):
|
|
448
|
+
def import_update(self, files_sha256: List[Tuple[str, str]], source_name: str, default_classification=None):
|
|
533
449
|
raise NotImplementedError()
|
|
534
450
|
|
|
535
451
|
# Define how to prepare the output directory before being served, must return the path of the directory to serve.
|
|
@@ -588,7 +504,7 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
588
504
|
self.sleep(60)
|
|
589
505
|
continue
|
|
590
506
|
|
|
591
|
-
def serve_directory(self, new_directory: str, new_time: str
|
|
507
|
+
def serve_directory(self, new_directory: str, new_time: str):
|
|
592
508
|
self.log.info("Update finished with new data.")
|
|
593
509
|
new_tar = ''
|
|
594
510
|
|
|
@@ -600,8 +516,8 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
600
516
|
# Pull signature metadata from the API
|
|
601
517
|
signature_map = {
|
|
602
518
|
item['signature_id']: item
|
|
603
|
-
for item in
|
|
604
|
-
|
|
519
|
+
for item in self.datastore.signature.stream_search(query=self.signatures_query,
|
|
520
|
+
fl="classification,source,status,signature_id,name")
|
|
605
521
|
}
|
|
606
522
|
else:
|
|
607
523
|
# Pull source metadata from synced service configuration
|
|
@@ -691,32 +607,3 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
691
607
|
self.local_update_flag.set()
|
|
692
608
|
self.sleep(60)
|
|
693
609
|
continue
|
|
694
|
-
|
|
695
|
-
def ensure_service_account(self):
|
|
696
|
-
"""Check that the update service account exists, if it doesn't, create it."""
|
|
697
|
-
uname = 'update_service_account'
|
|
698
|
-
user_data = self.datastore.user.get_if_exists(uname)
|
|
699
|
-
if user_data:
|
|
700
|
-
if user_data.roles and user_data.roles == UPDATER_API_ROLES:
|
|
701
|
-
# User exists and has the expected roles, we're good to go
|
|
702
|
-
return uname
|
|
703
|
-
|
|
704
|
-
# User exist but has no roles, let's update the user's roles
|
|
705
|
-
user_data.type = ["custom"]
|
|
706
|
-
user_data.roles = UPDATER_API_ROLES
|
|
707
|
-
else:
|
|
708
|
-
# User does not exist, let's create the user
|
|
709
|
-
user_data = User({
|
|
710
|
-
"agrees_with_tos": "NOW",
|
|
711
|
-
"classification": classification.RESTRICTED,
|
|
712
|
-
"name": "Update Account",
|
|
713
|
-
"password": get_password_hash(''.join(random.choices(string.ascii_letters, k=20))),
|
|
714
|
-
"uname": uname,
|
|
715
|
-
"type": ["custom"],
|
|
716
|
-
"roles": UPDATER_API_ROLES
|
|
717
|
-
})
|
|
718
|
-
|
|
719
|
-
self.datastore.user.save(uname, user_data)
|
|
720
|
-
self.datastore.user_settings.save(uname, UserSettings())
|
|
721
|
-
|
|
722
|
-
return uname
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: assemblyline-v4-service
|
|
3
|
-
Version: 4.5.0.
|
|
3
|
+
Version: 4.5.0.dev7
|
|
4
4
|
Summary: Assemblyline 4 - Service base
|
|
5
5
|
Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
|
|
6
6
|
Author: CCCS Assemblyline development team
|
|
7
7
|
Author-email: assemblyline@cyber.gc.ca
|
|
8
8
|
License: MIT
|
|
9
9
|
Keywords: assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs
|
|
10
|
-
Platform: UNKNOWN
|
|
11
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
12
11
|
Classifier: Intended Audience :: Developers
|
|
13
12
|
Classifier: Topic :: Software Development :: Libraries
|
|
@@ -29,7 +28,6 @@ Requires-Dist: pillow ==10.2.0
|
|
|
29
28
|
Requires-Dist: python-Levenshtein
|
|
30
29
|
Requires-Dist: regex
|
|
31
30
|
Provides-Extra: updater
|
|
32
|
-
Requires-Dist: assemblyline-client ; extra == 'updater'
|
|
33
31
|
Requires-Dist: gunicorn[gevent] ; extra == 'updater'
|
|
34
32
|
Requires-Dist: flask ; extra == 'updater'
|
|
35
33
|
Requires-Dist: gitpython ; extra == 'updater'
|
|
@@ -270,5 +268,3 @@ To test an Assemblyline service in standalone mode, the [run_service_once.py](ht
|
|
|
270
268
|
|
|
271
269
|
|
|
272
270
|
3. The `results.json` and any extracted/supplementary files will be outputted to `/home/ubuntu/testfile_resultsample`
|
|
273
|
-
|
|
274
|
-
|
{assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/RECORD
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
assemblyline_v4_service/VERSION,sha256=
|
|
1
|
+
assemblyline_v4_service/VERSION,sha256=7ZEyNKxSJLFtICSssiQ0qITaqdemo6PVzpK4BEIon4g,11
|
|
2
2
|
assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
|
|
4
4
|
assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -19,10 +19,10 @@ assemblyline_v4_service/dev/run_service_once.py,sha256=4K3ljw0MnfPGw0-6lzc_vtUYg
|
|
|
19
19
|
assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
|
|
21
21
|
assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
|
|
22
|
-
assemblyline_v4_service/updater/client.py,sha256=
|
|
22
|
+
assemblyline_v4_service/updater/client.py,sha256=HNmBKJI9VXsRso8IG3JojWsk9ujXJUu_cSJmKh7oG_k,4772
|
|
23
23
|
assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
|
|
24
24
|
assemblyline_v4_service/updater/helper.py,sha256=HbH5p6UTdHyIgoctF1c1pQkoqTtzaxfHOi9KXGwn0eM,9435
|
|
25
|
-
assemblyline_v4_service/updater/updater.py,sha256=
|
|
25
|
+
assemblyline_v4_service/updater/updater.py,sha256=ze6Ja7AQYLt13e67lb_tdfenUw0k_DNDe9n6Qx1BFNM,28933
|
|
26
26
|
test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
27
|
test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
|
|
28
28
|
test/test_run_privileged_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
|
|
@@ -37,8 +37,8 @@ test/test_common/test_request.py,sha256=wxSwnOj-_YOv2SuZjOJsw09q8A7p8GJmJuK4vozq
|
|
|
37
37
|
test/test_common/test_result.py,sha256=Wm0Cs5kZRzlZr0jL-l8OTsYAvkoN2eaB3NkeXzvyssI,42208
|
|
38
38
|
test/test_common/test_task.py,sha256=jnfF68EgJIu30Pz_4jiJHkncfI-3XpGaut5r79KIXOA,18718
|
|
39
39
|
test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
|
|
40
|
-
assemblyline_v4_service-4.5.0.
|
|
41
|
-
assemblyline_v4_service-4.5.0.
|
|
42
|
-
assemblyline_v4_service-4.5.0.
|
|
43
|
-
assemblyline_v4_service-4.5.0.
|
|
44
|
-
assemblyline_v4_service-4.5.0.
|
|
40
|
+
assemblyline_v4_service-4.5.0.dev7.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
|
|
41
|
+
assemblyline_v4_service-4.5.0.dev7.dist-info/METADATA,sha256=LTQhp1Wb3tGNBx-lhgi8Sd0MCDaxYSsog2yfA_xiGug,9738
|
|
42
|
+
assemblyline_v4_service-4.5.0.dev7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
43
|
+
assemblyline_v4_service-4.5.0.dev7.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
|
|
44
|
+
assemblyline_v4_service-4.5.0.dev7.dist-info/RECORD,,
|
|
File without changes
|
{assemblyline_v4_service-4.5.0.dev5.dist-info → assemblyline_v4_service-4.5.0.dev7.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|