assemblyline-v4-service 4.5.0.dev5__py3-none-any.whl → 4.5.0.dev7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-v4-service might be problematic. Click here for more details.

@@ -1 +1 @@
1
- 4.5.0.dev5
1
+ 4.5.0.dev7
@@ -1,24 +1,22 @@
1
1
  import os
2
2
 
3
- from assemblyline.common import forge
4
3
  from assemblyline.odm.models.signature import Signature as SignatureModel
5
- from assemblyline_client import Client4, get_client as get_AL_client
6
- from assemblyline_client.v4_client.module.signature import Signature as SignatureAPI
4
+ from assemblyline_core.badlist_client import BadlistClient
5
+ from assemblyline_core.safelist_client import SafelistClient
6
+ from assemblyline_core.signature_client import SignatureClient
7
7
 
8
8
  from typing import Any, Dict, List, Union
9
9
 
10
10
  SIGNATURE_UPDATE_BATCH = int(os.environ.get('SIGNATURE_UPDATE_BATCH', '1000'))
11
11
 
12
12
 
13
- class Signature(SignatureAPI):
14
- def __init__(self, connection, datastore=None):
15
- super().__init__(connection)
16
- self.datastore = datastore
17
- if not datastore:
18
- self.datastore = forge.get_datastore()
13
+ class SyncableSignature(SignatureClient):
14
+ def __init__(self, datastore, config=None):
15
+ super().__init__(datastore, config)
19
16
  self.sync = False
20
17
 
21
- def add_update_many(self, source: str, sig_type: str, data: List[Union[dict, SignatureModel]], dedup_name: bool = True) -> Dict[str, Any]:
18
+ def add_update_many(self, source: str, sig_type: str, data: List[Union[dict, SignatureModel]],
19
+ dedup_name: bool = True) -> Dict[str, Any]:
22
20
  # This version of the API allows to sync signatures with the system by making direct changes to the datastore
23
21
  # Signatures that no longer exist at the source will be DISABLED to maintain active synchronicity,
24
22
  # but users can always re-deploy signatures if desired
@@ -51,18 +49,19 @@ class Signature(SignatureAPI):
51
49
  # Get the list of signatures that currently existing in the system for the source
52
50
  existing_signature_ids = set([
53
51
  i.id for i in self.datastore.signature.stream_search(f"source:{source} AND type:{sig_type}", fl='id')
54
- ])
52
+ ])
55
53
 
56
54
  # Find the signature IDs that don't exist at this source anymore and disable them
57
55
  for missing_signature_id in (existing_signature_ids - current_signature_ids):
58
56
  missing_signature = self.datastore.signature.get(missing_signature_id)
59
- if missing_signature.state_change_user in ['update_service_account', None] and missing_signature.status != 'DISABLED':
57
+ if missing_signature.state_change_user in ['update_service_account', None] and \
58
+ missing_signature.status != 'DISABLED':
60
59
  # Only disable signature if it doesn't seem to be in use/altered by a (real) user
61
60
  self.datastore.signature.update(missing_signature_id,
62
61
  [(self.datastore.signature.UPDATE_SET, 'status', 'DISABLED'),
63
62
  (self.datastore.signature.UPDATE_SET, 'last_modified', 'NOW')])
64
63
 
65
- # Proceed with adding/updating signatures via the API server
64
+ # Proceed with adding/updating signatures
66
65
  if len(data) < SIGNATURE_UPDATE_BATCH:
67
66
  # Update all of them in a single batch
68
67
  return super().add_update_many(source, sig_type, data, dedup_name)
@@ -81,23 +80,19 @@ class Signature(SignatureAPI):
81
80
 
82
81
  # Split up data into batches to avoid server timeouts handling requests
83
82
  batch_num = 0
84
- start = batch_num*SIGNATURE_UPDATE_BATCH
83
+ start = batch_num * SIGNATURE_UPDATE_BATCH
85
84
  while start < len(data):
86
- end = (batch_num+1)*SIGNATURE_UPDATE_BATCH
85
+ end = (batch_num + 1) * SIGNATURE_UPDATE_BATCH
87
86
  update_response(super().add_update_many(source, sig_type, data[start:end], dedup_name))
88
87
  batch_num += 1
89
- start = batch_num*SIGNATURE_UPDATE_BATCH
88
+ start = batch_num * SIGNATURE_UPDATE_BATCH
90
89
 
91
90
  return response
92
91
 
93
92
 
94
- def get_client(server, auth=None, cert=None, debug=lambda x: None, headers=None, retries=0,
95
- silence_requests_warnings=True, apikey=None, verify=True, timeout=None, oauth=None,
96
- datastore=None) -> Client4:
97
-
98
- client = get_AL_client(server, auth, cert, debug, headers,
99
- retries, silence_requests_warnings,
100
- apikey, verify, timeout, oauth)
101
- # Override Signature module with custom implementation
102
- client.signature = Signature(client._connection, datastore=datastore)
103
- return client
93
+ class UpdaterClient(object):
94
+ def __init__(self, datastore) -> None:
95
+ self.datastore = datastore
96
+ self.badlist = BadlistClient(datastore)
97
+ self.safelist = SafelistClient(datastore)
98
+ self.signature = SyncableSignature(datastore)
@@ -7,34 +7,24 @@ import logging
7
7
  import time
8
8
  import json
9
9
  import tempfile
10
- import string
11
- import random
12
10
  import tarfile
13
11
  import threading
14
12
  import subprocess
15
13
  import hashlib
16
- from contextlib import contextmanager
17
14
  from queue import Queue
18
- from zipfile import ZipFile, BadZipFile
15
+ from zipfile import ZipFile
19
16
 
20
17
  from assemblyline.common import forge, log as al_log
21
18
  from assemblyline.common.isotime import epoch_to_iso, now_as_iso
22
- from assemblyline.common.identify import zip_ident
23
- from assemblyline.common.security import get_password_hash
24
19
  from assemblyline.odm.messages.changes import Operation, ServiceChange, SignatureChange
25
20
  from assemblyline.remote.datatypes.events import EventSender, EventWatcher
26
21
 
27
22
  from assemblyline_core.server_base import ThreadedCoreBase, ServiceStage
28
23
  from assemblyline.odm.models.service import Service, UpdateSource
29
24
  from assemblyline.remote.datatypes.hash import Hash
30
- from assemblyline.common.security import get_random_password, get_password_hash
31
- from assemblyline.remote.datatypes.lock import Lock
32
- from assemblyline.odm.models.user import User
33
- from assemblyline.odm.models.user_settings import UserSettings
34
25
 
35
- from assemblyline_client import Client4
36
26
  from assemblyline_v4_service.common.base import SIGNATURES_META_FILENAME
37
- from assemblyline_v4_service.updater.client import get_client
27
+ from assemblyline_v4_service.updater.client import UpdaterClient
38
28
  from assemblyline_v4_service.updater.helper import url_download, git_clone_repo, SkipSource, filter_downloads
39
29
 
40
30
  if typing.TYPE_CHECKING:
@@ -53,8 +43,6 @@ SOURCE_EXTRA_KEY = 'source_extra'
53
43
  SOURCE_STATUS_KEY = 'status'
54
44
  SOURCE_UPDATE_ATTEMPT_DELAY_BASE = int(os.environ.get("SOURCE_UPDATE_ATTEMPT_DELAY_BASE", "5"))
55
45
  SOURCE_UPDATE_ATTEMPT_MAX_RETRY = int(os.environ.get("SOURCE_UPDATE_ATTEMPT_MAX_RETRY", "3"))
56
- UI_SERVER = os.getenv('UI_SERVER', 'https://nginx')
57
- UI_SERVER_ROOT_CA = os.environ.get('UI_SERVER_ROOT_CA', '/etc/assemblyline/ssl/al_root-ca.crt')
58
46
  UPDATER_DIR = os.getenv('UPDATER_DIR', os.path.join(tempfile.gettempdir(), 'updater'))
59
47
  UPDATER_API_ROLES = ['badlist_manage', 'signature_import', 'signature_download',
60
48
  'signature_view', 'safelist_manage', 'apikey_access', 'signature_manage']
@@ -63,29 +51,6 @@ STATUS_FILE = '/tmp/status'
63
51
  classification = forge.get_classification()
64
52
 
65
53
 
66
- @contextmanager
67
- def temporary_api_key(ds: AssemblylineDatastore, user_name: str, permissions=('R', 'W')):
68
- """Creates a context where a temporary API key is available."""
69
- with Lock(f'user-{user_name}', timeout=10):
70
- name = ''.join(random.choices(string.ascii_lowercase, k=20))
71
- random_pass = get_random_password(length=48)
72
- user = ds.user.get(user_name)
73
- user.apikeys[name] = {
74
- "password": get_password_hash(random_pass),
75
- "acl": permissions,
76
- "roles": UPDATER_API_ROLES
77
- }
78
- ds.user.save(user_name, user)
79
-
80
- try:
81
- yield f"{name}:{random_pass}"
82
- finally:
83
- with Lock(f'user-{user_name}', timeout=10):
84
- user = ds.user.get(user_name)
85
- user.apikeys.pop(name)
86
- ds.user.save(user_name, user)
87
-
88
-
89
54
  # A Queue derivative that respects uniqueness of items as well as order
90
55
  class UniqueQueue(Queue):
91
56
  # Put a new item in the queue
@@ -127,6 +92,7 @@ class ServiceUpdater(ThreadedCoreBase):
127
92
  self.event_sender = EventSender('changes.services',
128
93
  host=self.config.core.redis.nonpersistent.host,
129
94
  port=self.config.core.redis.nonpersistent.port)
95
+ self.client = UpdaterClient(self.datastore)
130
96
 
131
97
  self.service_change_watcher = EventWatcher(self.redis, deserializer=ServiceChange.deserialize)
132
98
  self.service_change_watcher.register(f'changes.services.{SERVICE_NAME}', self._handle_service_change_event)
@@ -164,9 +130,6 @@ class ServiceUpdater(ThreadedCoreBase):
164
130
  status_query = ' OR '.join([f'status:{s}' for s in self.statuses])
165
131
  self.signatures_query = f"type:{self.updater_type} AND ({status_query})"
166
132
 
167
- # SSL configuration to UI_SERVER
168
- self.verify = None if not os.path.exists(UI_SERVER_ROOT_CA) else UI_SERVER_ROOT_CA
169
-
170
133
  def trigger_update(self):
171
134
  self.source_update_flag.set()
172
135
 
@@ -349,177 +312,131 @@ class ServiceUpdater(ThreadedCoreBase):
349
312
  if not os.path.exists(UPDATER_DIR):
350
313
  os.makedirs(UPDATER_DIR)
351
314
 
352
- self.log.info("Setup service account.")
353
- username = self.ensure_service_account()
354
- self.log.info("Create temporary API key.")
355
- with temporary_api_key(self.datastore, username) as api_key:
356
- self.log.info(f"Connecting to Assemblyline API: {UI_SERVER}")
357
- al_client = get_client(UI_SERVER, apikey=(username, api_key), verify=self.verify, datastore=self.datastore)
358
-
359
- # Create a temporary file for the time keeper
360
- time_keeper = tempfile.NamedTemporaryFile(prefix="time_keeper_", dir=UPDATER_DIR, delete=False)
361
- time_keeper.close()
362
- time_keeper = time_keeper.name
363
-
364
- if self._service.update_config.generates_signatures:
365
- output_directory = tempfile.mkdtemp(prefix="update_dir_", dir=UPDATER_DIR)
366
-
367
- # Check if new signatures have been added
368
- self.log.info("Check for new signatures.")
369
- if al_client.signature.update_available(
370
- since=epoch_to_iso(old_update_time) or '', sig_type=self.updater_type)['update_available']:
371
- self.log.info("An update is available for download from the datastore")
372
-
373
- self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}")
374
-
375
- extracted_zip = False
376
- attempt = 0
377
-
378
- # Sometimes a zip file isn't always returned, will affect
379
- # service's use of signature source. Patience..
380
- while not extracted_zip and attempt < 5:
381
- temp_zip_file = os.path.join(output_directory, 'temp.zip')
382
- al_client.signature.download(output=temp_zip_file, query=self.signatures_query)
383
- self.log.debug(f"Downloading update to {temp_zip_file}")
384
- if os.path.exists(temp_zip_file) and os.path.getsize(temp_zip_file) > 0:
385
- self.log.debug(
386
- f"File type ({os.path.getsize(temp_zip_file)}B): {zip_ident(temp_zip_file, 'unknown')}")
387
- try:
388
- with ZipFile(temp_zip_file, 'r') as zip_f:
389
- zip_f.extractall(output_directory)
390
- extracted_zip = True
391
- self.log.info("Zip extracted.")
392
- except BadZipFile:
393
- attempt += 1
394
- self.log.warning(f"[{attempt}/5] Bad zip. Trying again after 30s...")
395
- time.sleep(30)
396
- except Exception as e:
397
- self.log.error(f'Problem while extracting signatures to disk: {e}')
398
- break
399
-
400
- os.remove(temp_zip_file)
401
-
402
- if extracted_zip:
403
- self.log.info("New ruleset successfully downloaded and ready to use")
404
- self.serve_directory(output_directory, time_keeper, al_client)
405
- else:
406
- self.log.error("Signatures aren't saved to disk.")
407
- shutil.rmtree(output_directory, ignore_errors=True)
408
- if os.path.exists(time_keeper):
409
- os.unlink(time_keeper)
410
- else:
411
- self.log.info("No signature updates available.")
412
- shutil.rmtree(output_directory, ignore_errors=True)
413
- if os.path.exists(time_keeper):
414
- os.unlink(time_keeper)
315
+ # Create a temporary file for the time keeper
316
+ time_keeper = tempfile.NamedTemporaryFile(prefix="time_keeper_", dir=UPDATER_DIR, delete=False)
317
+ time_keeper.close()
318
+ time_keeper = time_keeper.name
319
+
320
+ if self._service.update_config.generates_signatures:
321
+ output_directory = tempfile.mkdtemp(prefix="update_dir_", dir=UPDATER_DIR)
322
+
323
+ # Check if new signatures have been added
324
+ self.log.info("Check for new signatures.")
325
+ if self.client.signature.update_available(since=epoch_to_iso(old_update_time) or None,
326
+ sig_type=self.updater_type):
327
+ self.log.info("An update is available for download from the datastore")
328
+
329
+ self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}")
330
+
331
+ with ZipFile(self.client.signature.download(self.signatures_query), 'r') as zip_f:
332
+ zip_f.extractall(output_directory)
333
+ self.log.info("New ruleset successfully downloaded and ready to use")
334
+ self.serve_directory(output_directory, time_keeper)
415
335
  else:
416
- output_directory = self.prepare_output_directory()
417
- self.serve_directory(output_directory, time_keeper, al_client)
336
+ self.log.info("No signature updates available.")
337
+ shutil.rmtree(output_directory, ignore_errors=True)
338
+ if os.path.exists(time_keeper):
339
+ os.unlink(time_keeper)
340
+ else:
341
+ output_directory = self.prepare_output_directory()
342
+ self.serve_directory(output_directory, time_keeper)
418
343
 
419
344
  def do_source_update(self, service: Service) -> None:
420
- self.log.info(f"Connecting to Assemblyline API: {UI_SERVER}...")
421
345
  run_time = time.time()
422
- username = self.ensure_service_account()
423
- with temporary_api_key(self.datastore, username) as api_key:
424
- with tempfile.TemporaryDirectory() as update_dir:
425
- al_client = get_client(
426
- UI_SERVER, apikey=(username, api_key),
427
- verify=self.verify, datastore=self.datastore)
428
- self.log.info("Connected!")
429
-
430
- # Parse updater configuration
431
- previous_hashes: dict[str, dict[str, str]] = self.get_source_extra()
432
- sources: dict[str, UpdateSource] = {_s['name']: _s for _s in service.update_config.sources}
433
- files_sha256: dict[str, dict[str, str]] = {}
434
-
435
- # Map already visited URIs to download paths (avoid re-cloning/re-downloads)
436
- seen_fetches = dict()
437
-
438
- # Go through each source queued and download file
439
- while self.update_queue.qsize():
440
- update_attempt = -1
441
- source_name = self.update_queue.get()
442
- while update_attempt < SOURCE_UPDATE_ATTEMPT_MAX_RETRY:
443
- # Introduce an exponential delay between each attempt
444
- time.sleep(SOURCE_UPDATE_ATTEMPT_DELAY_BASE**update_attempt)
445
- update_attempt += 1
446
-
447
- # Set current source for pushing state to UI
448
- self._current_source = source_name
449
- source_obj = sources[source_name]
450
- old_update_time = self.get_source_update_time()
451
-
452
- self.push_status("UPDATING", "Starting..")
453
- source = source_obj.as_primitives()
454
- uri: str = source['uri']
455
- default_classification = source.get('default_classification', classification.UNRESTRICTED)
456
- # Enable signature syncing if the source specifies it
457
- al_client.signature.sync = source.get('sync', False)
458
-
459
- try:
460
- self.push_status("UPDATING", "Pulling..")
461
- output = None
462
- seen_fetch = seen_fetches.get(uri)
463
- if seen_fetch == 'skipped':
464
- # Skip source if another source says nothing has changed
465
- raise SkipSource
466
- elif seen_fetch and os.path.exists(seen_fetch):
467
- # We've already fetched something from the same URI, re-use downloaded path
468
- self.log.info(f'Already visited {uri} in this run. Using cached download path..')
469
- output = seen_fetches[uri]
470
- else:
471
- # Pull sources from external locations (method depends on the URL)
472
- try:
473
- # First we'll attempt by performing a Git clone
474
- # (since not all services hint at being a repository in their URL),
475
- output = git_clone_repo(source, old_update_time, self.log, update_dir)
476
- except SkipSource:
477
- raise
478
- except Exception as git_ex:
479
- # Should that fail, we'll attempt a direct-download using Python Requests
480
- if not uri.endswith('.git'):
481
- # Proceed with direct download, raise exception as required if necessary
482
- output = url_download(source, old_update_time, self.log, update_dir)
483
- else:
484
- # Raise Git Exception
485
- raise git_ex
486
- # Add output path to the list of seen fetches in this run
487
- seen_fetches[uri] = output
488
-
489
- files = filter_downloads(output, source['pattern'], self.default_pattern)
490
-
491
- # Add to collection of sources for caching purposes
492
- self.log.info(f"Found new {self.updater_type} rule files to process for {source_name}!")
493
- validated_files = list()
494
- for file, sha256 in files:
495
- files_sha256.setdefault(source_name, {})
496
- if previous_hashes.get(
497
- source_name, {}).get(
498
- file, None) != sha256 and self.is_valid(file):
499
- files_sha256[source_name][file] = sha256
500
- validated_files.append((file, sha256))
501
-
502
- self.push_status("UPDATING", "Importing..")
503
- # Import into Assemblyline
504
- self.import_update(validated_files, al_client, source_name, default_classification)
505
- self.push_status("DONE", "Signature(s) Imported.")
506
- except SkipSource:
507
- # This source hasn't changed, no need to re-import into Assemblyline
508
- self.log.info(f'No new {self.updater_type} rule files to process for {source_name}')
509
- if source_name in previous_hashes:
510
- files_sha256[source_name] = previous_hashes[source_name]
511
- seen_fetches[uri] = "skipped"
512
- self.push_status("DONE", "Skipped.")
513
- break
514
- except Exception as e:
515
- # There was an issue with this source, report and continue to the next
516
- self.log.error(f"Problem with {source['name']}: {e}")
517
- self.push_status("ERROR", str(e))
518
- continue
519
-
520
- self.set_source_update_time(run_time)
521
- self.set_source_extra(files_sha256)
346
+ with tempfile.TemporaryDirectory() as update_dir:
347
+ # Parse updater configuration
348
+ previous_hashes: dict[str, dict[str, str]] = self.get_source_extra()
349
+ sources: dict[str, UpdateSource] = {_s['name']: _s for _s in service.update_config.sources}
350
+ files_sha256: dict[str, dict[str, str]] = {}
351
+
352
+ # Map already visited URIs to download paths (avoid re-cloning/re-downloads)
353
+ seen_fetches = dict()
354
+
355
+ # Go through each source queued and download file
356
+ while self.update_queue.qsize():
357
+ update_attempt = -1
358
+ source_name = self.update_queue.get()
359
+ while update_attempt < SOURCE_UPDATE_ATTEMPT_MAX_RETRY:
360
+ # Introduce an exponential delay between each attempt
361
+ time.sleep(SOURCE_UPDATE_ATTEMPT_DELAY_BASE**update_attempt)
362
+ update_attempt += 1
363
+
364
+ # Set current source for pushing state to UI
365
+ self._current_source = source_name
366
+ source_obj = sources[source_name]
367
+ old_update_time = self.get_source_update_time()
368
+
369
+ self.push_status("UPDATING", "Starting..")
370
+ source = source_obj.as_primitives()
371
+ uri: str = source['uri']
372
+ default_classification = source.get('default_classification', classification.UNRESTRICTED)
373
+ # Enable signature syncing if the source specifies it
374
+ self.client.signature.sync = source.get('sync', False)
375
+
376
+ try:
377
+ self.push_status("UPDATING", "Pulling..")
378
+ output = None
379
+ seen_fetch = seen_fetches.get(uri)
380
+ if seen_fetch == 'skipped':
381
+ # Skip source if another source says nothing has changed
382
+ raise SkipSource
383
+ elif seen_fetch and os.path.exists(seen_fetch):
384
+ # We've already fetched something from the same URI, re-use downloaded path
385
+ self.log.info(f'Already visited {uri} in this run. Using cached download path..')
386
+ output = seen_fetches[uri]
387
+ else:
388
+ # Pull sources from external locations (method depends on the URL)
389
+ try:
390
+ # First we'll attempt by performing a Git clone
391
+ # (since not all services hint at being a repository in their URL),
392
+ output = git_clone_repo(source, old_update_time, self.log, update_dir)
393
+ except SkipSource:
394
+ raise
395
+ except Exception as git_ex:
396
+ # Should that fail, we'll attempt a direct-download using Python Requests
397
+ if not uri.endswith('.git'):
398
+ # Proceed with direct download, raise exception as required if necessary
399
+ output = url_download(source, old_update_time, self.log, update_dir)
400
+ else:
401
+ # Raise Git Exception
402
+ raise git_ex
403
+ # Add output path to the list of seen fetches in this run
404
+ seen_fetches[uri] = output
405
+
406
+ files = filter_downloads(output, source['pattern'], self.default_pattern)
407
+
408
+ # Add to collection of sources for caching purposes
409
+ self.log.info(f"Found new {self.updater_type} rule files to process for {source_name}!")
410
+ validated_files = list()
411
+ for file, sha256 in files:
412
+ files_sha256.setdefault(source_name, {})
413
+ if previous_hashes.get(
414
+ source_name, {}).get(
415
+ file, None) != sha256 and self.is_valid(file):
416
+ files_sha256[source_name][file] = sha256
417
+ validated_files.append((file, sha256))
418
+
419
+ self.push_status("UPDATING", "Importing..")
420
+ # Import into Assemblyline
421
+ self.import_update(validated_files, self.client, source_name, default_classification)
422
+ self.push_status("DONE", "Signature(s) Imported.")
423
+ except SkipSource:
424
+ # This source hasn't changed, no need to re-import into Assemblyline
425
+ self.log.info(f'No new {self.updater_type} rule files to process for {source_name}')
426
+ if source_name in previous_hashes:
427
+ files_sha256[source_name] = previous_hashes[source_name]
428
+ seen_fetches[uri] = "skipped"
429
+ self.push_status("DONE", "Skipped.")
522
430
  break
431
+ except Exception as e:
432
+ # There was an issue with this source, report and continue to the next
433
+ self.log.error(f"Problem with {source['name']}: {e}")
434
+ self.push_status("ERROR", str(e))
435
+ continue
436
+
437
+ self.set_source_update_time(run_time)
438
+ self.set_source_extra(files_sha256)
439
+ break
523
440
  self.set_active_config_hash(self.config_hash(service))
524
441
  self.local_update_flag.set()
525
442
 
@@ -528,8 +445,7 @@ class ServiceUpdater(ThreadedCoreBase):
528
445
  return True
529
446
 
530
447
  # Define how your source update gets imported into Assemblyline
531
- def import_update(self, files_sha256: List[Tuple[str, str]], client: Client4, source_name: str,
532
- default_classification=None):
448
+ def import_update(self, files_sha256: List[Tuple[str, str]], source_name: str, default_classification=None):
533
449
  raise NotImplementedError()
534
450
 
535
451
  # Define how to prepare the output directory before being served, must return the path of the directory to serve.
@@ -588,7 +504,7 @@ class ServiceUpdater(ThreadedCoreBase):
588
504
  self.sleep(60)
589
505
  continue
590
506
 
591
- def serve_directory(self, new_directory: str, new_time: str, client: Client4):
507
+ def serve_directory(self, new_directory: str, new_time: str):
592
508
  self.log.info("Update finished with new data.")
593
509
  new_tar = ''
594
510
 
@@ -600,8 +516,8 @@ class ServiceUpdater(ThreadedCoreBase):
600
516
  # Pull signature metadata from the API
601
517
  signature_map = {
602
518
  item['signature_id']: item
603
- for item in client.search.stream.signature(query=self.signatures_query,
604
- fl="classification,source,status,signature_id,name")
519
+ for item in self.datastore.signature.stream_search(query=self.signatures_query,
520
+ fl="classification,source,status,signature_id,name")
605
521
  }
606
522
  else:
607
523
  # Pull source metadata from synced service configuration
@@ -691,32 +607,3 @@ class ServiceUpdater(ThreadedCoreBase):
691
607
  self.local_update_flag.set()
692
608
  self.sleep(60)
693
609
  continue
694
-
695
- def ensure_service_account(self):
696
- """Check that the update service account exists, if it doesn't, create it."""
697
- uname = 'update_service_account'
698
- user_data = self.datastore.user.get_if_exists(uname)
699
- if user_data:
700
- if user_data.roles and user_data.roles == UPDATER_API_ROLES:
701
- # User exists and has the expected roles, we're good to go
702
- return uname
703
-
704
- # User exist but has no roles, let's update the user's roles
705
- user_data.type = ["custom"]
706
- user_data.roles = UPDATER_API_ROLES
707
- else:
708
- # User does not exist, let's create the user
709
- user_data = User({
710
- "agrees_with_tos": "NOW",
711
- "classification": classification.RESTRICTED,
712
- "name": "Update Account",
713
- "password": get_password_hash(''.join(random.choices(string.ascii_letters, k=20))),
714
- "uname": uname,
715
- "type": ["custom"],
716
- "roles": UPDATER_API_ROLES
717
- })
718
-
719
- self.datastore.user.save(uname, user_data)
720
- self.datastore.user_settings.save(uname, UserSettings())
721
-
722
- return uname
@@ -1,13 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-v4-service
3
- Version: 4.5.0.dev5
3
+ Version: 4.5.0.dev7
4
4
  Summary: Assemblyline 4 - Service base
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
6
6
  Author: CCCS Assemblyline development team
7
7
  Author-email: assemblyline@cyber.gc.ca
8
8
  License: MIT
9
9
  Keywords: assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs
10
- Platform: UNKNOWN
11
10
  Classifier: Development Status :: 5 - Production/Stable
12
11
  Classifier: Intended Audience :: Developers
13
12
  Classifier: Topic :: Software Development :: Libraries
@@ -29,7 +28,6 @@ Requires-Dist: pillow ==10.2.0
29
28
  Requires-Dist: python-Levenshtein
30
29
  Requires-Dist: regex
31
30
  Provides-Extra: updater
32
- Requires-Dist: assemblyline-client ; extra == 'updater'
33
31
  Requires-Dist: gunicorn[gevent] ; extra == 'updater'
34
32
  Requires-Dist: flask ; extra == 'updater'
35
33
  Requires-Dist: gitpython ; extra == 'updater'
@@ -270,5 +268,3 @@ To test an Assemblyline service in standalone mode, the [run_service_once.py](ht
270
268
 
271
269
 
272
270
  3. The `results.json` and any extracted/supplementary files will be outputted to `/home/ubuntu/testfile_resultsample`
273
-
274
-
@@ -1,4 +1,4 @@
1
- assemblyline_v4_service/VERSION,sha256=TpZxi8mvfrYGmMNvWFEERfJhJKdtTz3vlH09s-1Gjps,11
1
+ assemblyline_v4_service/VERSION,sha256=7ZEyNKxSJLFtICSssiQ0qITaqdemo6PVzpK4BEIon4g,11
2
2
  assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
3
  assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
4
4
  assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,10 +19,10 @@ assemblyline_v4_service/dev/run_service_once.py,sha256=4K3ljw0MnfPGw0-6lzc_vtUYg
19
19
  assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
20
  assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
21
21
  assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
22
- assemblyline_v4_service/updater/client.py,sha256=VKmkiDz3klvMzXiGSQTH7QN-MfHuJnwaIR7BaXy9aZM,5156
22
+ assemblyline_v4_service/updater/client.py,sha256=HNmBKJI9VXsRso8IG3JojWsk9ujXJUu_cSJmKh7oG_k,4772
23
23
  assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
24
24
  assemblyline_v4_service/updater/helper.py,sha256=HbH5p6UTdHyIgoctF1c1pQkoqTtzaxfHOi9KXGwn0eM,9435
25
- assemblyline_v4_service/updater/updater.py,sha256=V0pFwevPYf2ZMfhLajIRP2S-DanHx-ZDK89SmUNR_Fc,34701
25
+ assemblyline_v4_service/updater/updater.py,sha256=ze6Ja7AQYLt13e67lb_tdfenUw0k_DNDe9n6Qx1BFNM,28933
26
26
  test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
27
  test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
28
28
  test/test_run_privileged_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -37,8 +37,8 @@ test/test_common/test_request.py,sha256=wxSwnOj-_YOv2SuZjOJsw09q8A7p8GJmJuK4vozq
37
37
  test/test_common/test_result.py,sha256=Wm0Cs5kZRzlZr0jL-l8OTsYAvkoN2eaB3NkeXzvyssI,42208
38
38
  test/test_common/test_task.py,sha256=jnfF68EgJIu30Pz_4jiJHkncfI-3XpGaut5r79KIXOA,18718
39
39
  test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
40
- assemblyline_v4_service-4.5.0.dev5.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
41
- assemblyline_v4_service-4.5.0.dev5.dist-info/METADATA,sha256=zL4_Jm0A0Fl__y_TNckFHM_Z_gYFG0OYFOymlV7eik4,9814
42
- assemblyline_v4_service-4.5.0.dev5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
43
- assemblyline_v4_service-4.5.0.dev5.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
44
- assemblyline_v4_service-4.5.0.dev5.dist-info/RECORD,,
40
+ assemblyline_v4_service-4.5.0.dev7.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
41
+ assemblyline_v4_service-4.5.0.dev7.dist-info/METADATA,sha256=LTQhp1Wb3tGNBx-lhgi8Sd0MCDaxYSsog2yfA_xiGug,9738
42
+ assemblyline_v4_service-4.5.0.dev7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
43
+ assemblyline_v4_service-4.5.0.dev7.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
44
+ assemblyline_v4_service-4.5.0.dev7.dist-info/RECORD,,