assemblyline-core 4.5.0.dev5__tar.gz → 4.5.0.dev7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/PKG-INFO +1 -4
  2. assemblyline-core-4.5.0.dev7/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/badlist_client.py +113 -1
  4. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/safelist_client.py +132 -3
  5. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/scaler_server.py +1 -2
  6. assemblyline-core-4.5.0.dev7/assemblyline_core/signature_client.py +210 -0
  7. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/updater/run_updater.py +1 -2
  8. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core.egg-info/PKG-INFO +1 -4
  9. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core.egg-info/SOURCES.txt +4 -0
  10. assemblyline-core-4.5.0.dev7/test/test_badlist_client.py +291 -0
  11. assemblyline-core-4.5.0.dev7/test/test_safelist_client.py +326 -0
  12. assemblyline-core-4.5.0.dev7/test/test_signature_client.py +105 -0
  13. assemblyline-core-4.5.0.dev5/assemblyline_core/VERSION +0 -1
  14. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/LICENCE.md +0 -0
  15. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/README.md +0 -0
  16. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/__init__.py +0 -0
  17. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/alerter/__init__.py +0 -0
  18. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/alerter/processing.py +0 -0
  19. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/alerter/run_alerter.py +0 -0
  20. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/archiver/__init__.py +0 -0
  21. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/archiver/run_archiver.py +0 -0
  22. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/dispatching/__init__.py +0 -0
  23. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/dispatching/__main__.py +0 -0
  24. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/dispatching/client.py +0 -0
  25. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/dispatching/dispatcher.py +0 -0
  26. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/dispatching/schedules.py +0 -0
  27. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/dispatching/timeout.py +0 -0
  28. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/expiry/__init__.py +0 -0
  29. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/expiry/run_expiry.py +0 -0
  30. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/ingester/__init__.py +0 -0
  31. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/ingester/__main__.py +0 -0
  32. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/ingester/constants.py +0 -0
  33. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/ingester/ingester.py +0 -0
  34. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/__init__.py +0 -0
  35. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/es_metrics.py +0 -0
  36. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  37. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/helper.py +0 -0
  38. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/metrics_server.py +0 -0
  39. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  40. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  41. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  42. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/plumber/__init__.py +0 -0
  43. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/plumber/run_plumber.py +0 -0
  44. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/__init__.py +0 -0
  45. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/client.py +0 -0
  46. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/creator/__init__.py +0 -0
  47. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/creator/run.py +0 -0
  48. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/creator/run_worker.py +0 -0
  49. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/loader/__init__.py +0 -0
  50. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/loader/run.py +0 -0
  51. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/loader/run_worker.py +0 -0
  52. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/replay/replay.py +0 -0
  53. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/__init__.py +0 -0
  54. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/collection.py +0 -0
  55. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  56. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  57. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/controllers/interface.py +0 -0
  58. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  59. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/scaler/run_scaler.py +0 -0
  60. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/server_base.py +0 -0
  61. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/submission_client.py +0 -0
  62. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/tasking_client.py +0 -0
  63. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/updater/__init__.py +0 -0
  64. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/updater/helper.py +0 -0
  65. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/vacuum/__init__.py +0 -0
  66. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/vacuum/crawler.py +0 -0
  67. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/vacuum/department_map.py +0 -0
  68. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/vacuum/safelist.py +0 -0
  69. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/vacuum/stream_map.py +0 -0
  70. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/vacuum/worker.py +0 -0
  71. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/workflow/__init__.py +0 -0
  72. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core/workflow/run_workflow.py +0 -0
  73. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  74. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core.egg-info/requires.txt +0 -0
  75. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/assemblyline_core.egg-info/top_level.txt +0 -0
  76. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/setup.cfg +0 -0
  77. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/setup.py +0 -0
  78. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_alerter.py +0 -0
  79. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_dispatcher.py +0 -0
  80. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_expiry.py +0 -0
  81. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_plumber.py +0 -0
  82. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_replay.py +0 -0
  83. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_scaler.py +0 -0
  84. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_scheduler.py +0 -0
  85. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.0.dev5 → assemblyline-core-4.5.0.dev7}/test/test_worker_submit.py +0 -0
@@ -1,13 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.dev5
3
+ Version: 4.5.0.dev7
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
7
7
  Author-email: assemblyline@cyber.gc.ca
8
8
  License: MIT
9
9
  Keywords: assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs
10
- Platform: UNKNOWN
11
10
  Classifier: Development Status :: 5 - Production/Stable
12
11
  Classifier: Intended Audience :: Developers
13
12
  Classifier: Topic :: Software Development :: Libraries
@@ -59,5 +58,3 @@ Make sure the different services get their latest update files.
59
58
  ##### Workflow
60
59
 
61
60
  Run the different workflows in the system and apply their labels, priority and status.
62
-
63
-
@@ -0,0 +1 @@
1
+ 4.5.0.dev7
@@ -5,6 +5,9 @@ from assemblyline.common import forge
5
5
  from assemblyline.common.chunk import chunk
6
6
  from assemblyline.common.isotime import now_as_iso
7
7
  from assemblyline.datastore.helper import AssemblylineDatastore
8
+ from assemblyline.odm.models.user import ROLES
9
+ from assemblyline.remote.datatypes.lock import Lock
10
+
8
11
 
9
12
  CHUNK_SIZE = 1000
10
13
  CLASSIFICATION = forge.get_classification()
@@ -14,6 +17,7 @@ class InvalidBadhash(Exception):
14
17
  pass
15
18
 
16
19
 
20
+ # Badlist class
17
21
  class BadlistClient:
18
22
  """A helper class to simplify badlisting for privileged services and service-server."""
19
23
 
@@ -22,7 +26,115 @@ class BadlistClient:
22
26
  self.config = config or forge.CachedObject(forge.get_config)
23
27
  self.datastore = datastore or forge.get_datastore(self.config)
24
28
 
25
- # Badlist
29
+ def _preprocess_object(self, data: dict) -> str:
30
+ # Set defaults
31
+ data.setdefault('classification', CLASSIFICATION.UNRESTRICTED)
32
+ data.setdefault('hashes', {})
33
+ data.setdefault('expiry_ts', None)
34
+ if data['type'] == 'tag':
35
+ # Remove file related fields
36
+ data.pop('file', None)
37
+ data.pop('hashes', None)
38
+
39
+ tag_data = data.get('tag', None)
40
+ if tag_data is None or 'type' not in tag_data or 'value' not in tag_data:
41
+ raise ValueError("Tag data not found")
42
+
43
+ hashed_value = f"{tag_data['type']}: {tag_data['value']}".encode('utf8')
44
+ data['hashes'] = {
45
+ 'md5': hashlib.md5(hashed_value).hexdigest(),
46
+ 'sha1': hashlib.sha1(hashed_value).hexdigest(),
47
+ 'sha256': hashlib.sha256(hashed_value).hexdigest()
48
+ }
49
+
50
+ elif data['type'] == 'file':
51
+ data.pop('tag', None)
52
+ data.setdefault('file', {})
53
+
54
+ # Ensure expiry_ts is set on tag-related items
55
+ dtl = data.pop('dtl', None) or self.config.core.expiry.badlisted_tag_dtl * 24 * 3600
56
+ if dtl:
57
+ data['expiry_ts'] = now_as_iso(dtl)
58
+
59
+ # Set last updated
60
+ data['added'] = data['updated'] = now_as_iso()
61
+
62
+ # Find the best hash to use for the key
63
+ for hash_key in ['sha256', 'sha1', 'md5']:
64
+ qhash = data['hashes'].get(hash_key, None)
65
+ if qhash:
66
+ break
67
+
68
+ # Validate hash length
69
+ if not qhash:
70
+ raise ValueError("No valid hash found")
71
+
72
+ return qhash
73
+
74
+ def add_update(self, badlist_object: dict, user: dict = None):
75
+ qhash = self._preprocess_object(badlist_object)
76
+
77
+ # Validate sources
78
+ src_map = {}
79
+ for src in badlist_object['sources']:
80
+ if user:
81
+ if src['type'] == 'user':
82
+ if src['name'] != user['uname']:
83
+ raise ValueError(f"You cannot add a source for another user. {src['name']} != {user['uname']}")
84
+ else:
85
+ if ROLES.signature_import not in user['roles']:
86
+ raise PermissionError("You do not have sufficient priviledges to add an external source.")
87
+
88
+ # Find the highest classification of all sources
89
+ badlist_object['classification'] = CLASSIFICATION.max_classification(
90
+ badlist_object['classification'], src.get('classification', None))
91
+
92
+ src_map[src['name']] = src
93
+
94
+ with Lock(f'add_or_update-badlist-{qhash}', 30):
95
+ old = self.datastore.badlist.get_if_exists(qhash, as_obj=False)
96
+ if old:
97
+ # Save data to the DB
98
+ self.datastore.badlist.save(qhash, BadlistClient._merge_hashes(badlist_object, old))
99
+ return qhash, "update"
100
+ else:
101
+ try:
102
+ badlist_object['sources'] = list(src_map.values())
103
+ self.datastore.badlist.save(qhash, badlist_object)
104
+ return qhash, "add"
105
+ except Exception as e:
106
+ return ValueError(f"Invalid data provided: {str(e)}")
107
+
108
+ def add_update_many(self, list_of_badlist_objects: list):
109
+ if not isinstance(list_of_badlist_objects, list):
110
+ raise ValueError("Could not get the list of hashes")
111
+
112
+ new_data = {}
113
+ for badlist_object in list_of_badlist_objects:
114
+ qhash = self._preprocess_object(badlist_object)
115
+ new_data[qhash] = badlist_object
116
+
117
+ # Get already existing hashes
118
+ old_data = self.datastore.badlist.multiget(list(new_data.keys()), as_dictionary=True, as_obj=False,
119
+ error_on_missing=False)
120
+
121
+ # Test signature names
122
+ plan = self.datastore.badlist.get_bulk_plan()
123
+ for key, val in new_data.items():
124
+ # Use maximum classification
125
+ old_val = old_data.get(key, {'classification': CLASSIFICATION.UNRESTRICTED, 'attribution': {},
126
+ 'hashes': {}, 'sources': [], 'type': val['type']})
127
+
128
+ # Add upsert operation
129
+ plan.add_upsert_operation(key, BadlistClient._merge_hashes(val, old_val))
130
+
131
+ if not plan.empty:
132
+ # Execute plan
133
+ res = self.datastore.badlist.bulk(plan)
134
+ return {"success": len(res['items']), "errors": res['errors']}
135
+
136
+ return {"success": 0, "errors": []}
137
+
26
138
  def exists(self, qhash):
27
139
  return self.datastore.badlist.get_if_exists(qhash, as_obj=False)
28
140
 
@@ -1,10 +1,12 @@
1
+ import hashlib
1
2
  import logging
2
-
3
3
  import yaml
4
4
 
5
5
  from assemblyline.common import forge
6
6
  from assemblyline.common.isotime import now_as_iso
7
7
  from assemblyline.datastore.helper import AssemblylineDatastore
8
+ from assemblyline.odm.models.user import ROLES
9
+ from assemblyline.remote.datatypes.lock import Lock
8
10
 
9
11
  CLASSIFICATION = forge.get_classification()
10
12
 
@@ -13,7 +15,7 @@ class InvalidSafehash(Exception):
13
15
  pass
14
16
 
15
17
 
16
- # Tasking class
18
+ # Safelist class
17
19
  class SafelistClient:
18
20
  """A helper class to simplify safelisting for privileged services and service-server."""
19
21
 
@@ -22,7 +24,134 @@ class SafelistClient:
22
24
  self.config = config or forge.CachedObject(forge.get_config)
23
25
  self.datastore = datastore or forge.get_datastore(self.config)
24
26
 
25
- # Safelist
27
+ def _preprocess_object(self, data: dict):
28
+ # Set defaults
29
+ data.setdefault('classification', CLASSIFICATION.UNRESTRICTED)
30
+ data.setdefault('hashes', {})
31
+ data.setdefault('expiry_ts', None)
32
+ if data['type'] == 'tag':
33
+ # Remove file related fields
34
+ data.pop('file', None)
35
+ data.pop('hashes', None)
36
+ data.pop('signature', None)
37
+
38
+ tag_data = data.get('tag', None)
39
+ if tag_data is None or 'type' not in tag_data or 'value' not in tag_data:
40
+ raise ValueError("Tag data not found")
41
+
42
+ hashed_value = f"{tag_data['type']}: {tag_data['value']}".encode('utf8')
43
+ data['hashes'] = {
44
+ 'md5': hashlib.md5(hashed_value).hexdigest(),
45
+ 'sha1': hashlib.sha1(hashed_value).hexdigest(),
46
+ 'sha256': hashlib.sha256(hashed_value).hexdigest()
47
+ }
48
+
49
+ elif data['type'] == 'signature':
50
+ # Remove file related fields
51
+ data.pop('file', None)
52
+ data.pop('hashes', None)
53
+ data.pop('tag', None)
54
+
55
+ sig_data = data.get('signature', None)
56
+ if sig_data is None or 'name' not in sig_data:
57
+ raise ValueError("Signature data not found")
58
+
59
+ hashed_value = f"signature: {sig_data['name']}".encode('utf8')
60
+ data['hashes'] = {
61
+ 'md5': hashlib.md5(hashed_value).hexdigest(),
62
+ 'sha1': hashlib.sha1(hashed_value).hexdigest(),
63
+ 'sha256': hashlib.sha256(hashed_value).hexdigest()
64
+ }
65
+
66
+ elif data['type'] == 'file':
67
+ data.pop('signature', None)
68
+ data.pop('tag', None)
69
+ data.setdefault('file', {})
70
+
71
+ # Ensure expiry_ts is set on tag-related items
72
+ dtl = data.pop('dtl', None) or self.config.core.expiry.safelisted_tag_dtl * 24 * 3600
73
+ if dtl:
74
+ data['expiry_ts'] = now_as_iso(dtl)
75
+
76
+ # Set last updated
77
+ data['added'] = data['updated'] = now_as_iso()
78
+
79
+ # Find the best hash to use for the key
80
+ for hash_key in ['sha256', 'sha1', 'md5']:
81
+ qhash = data['hashes'].get(hash_key, None)
82
+ if qhash:
83
+ break
84
+
85
+ # Validate hash length
86
+ if not qhash:
87
+ raise ValueError("No valid hash found")
88
+
89
+ return qhash
90
+
91
+ def add_update(self, safelist_object: dict, user: dict = None):
92
+ qhash = self._preprocess_object(safelist_object)
93
+
94
+ # Validate sources
95
+ src_map = {}
96
+ for src in safelist_object['sources']:
97
+ if user:
98
+ if src['type'] == 'user':
99
+ if src['name'] != user['uname']:
100
+ raise ValueError(f"You cannot add a source for another user. {src['name']} != {user['uname']}")
101
+ else:
102
+ if ROLES.signature_import not in user['roles']:
103
+ raise PermissionError("You do not have sufficient priviledges to add an external source.")
104
+
105
+ # Find the highest classification of all sources
106
+ safelist_object['classification'] = CLASSIFICATION.max_classification(
107
+ safelist_object['classification'], src.get('classification', None))
108
+
109
+ src_map[src['name']] = src
110
+
111
+ with Lock(f'add_or_update-safelist-{qhash}', 30):
112
+ old = self.datastore.safelist.get_if_exists(qhash, as_obj=False)
113
+ if old:
114
+ # Save data to the DB
115
+ self.datastore.safelist.save(qhash, SafelistClient._merge_hashes(safelist_object, old))
116
+ return qhash, "update"
117
+ else:
118
+ try:
119
+ safelist_object['sources'] = list(src_map.values())
120
+ self.datastore.safelist.save(qhash, safelist_object)
121
+ return qhash, "add"
122
+ except Exception as e:
123
+ return ValueError(f"Invalid data provided: {str(e)}")
124
+
125
+ def add_update_many(self, list_of_safelist_objects: list):
126
+ if not isinstance(list_of_safelist_objects, list):
127
+ raise ValueError("Could not get the list of hashes")
128
+
129
+ new_data = {}
130
+ for safelist_object in list_of_safelist_objects:
131
+ qhash = self._preprocess_object(safelist_object)
132
+ new_data[qhash] = safelist_object
133
+
134
+ # Get already existing hashes
135
+ old_data = self.datastore.safelist.multiget(list(new_data.keys()), as_dictionary=True, as_obj=False,
136
+ error_on_missing=False)
137
+
138
+ # Test signature names
139
+ plan = self.datastore.safelist.get_bulk_plan()
140
+ for key, val in new_data.items():
141
+ # Use maximum classification
142
+ old_val = old_data.get(key, {'classification': CLASSIFICATION.UNRESTRICTED,
143
+ 'hashes': {}, 'sources': [], 'type': val['type']})
144
+
145
+ # Add upsert operation
146
+ plan.add_upsert_operation(key, SafelistClient._merge_hashes(val, old_val))
147
+
148
+ if not plan.empty:
149
+ # Execute plan
150
+ res = self.datastore.safelist.bulk(plan)
151
+ return {"success": len(res['items']), "errors": res['errors']}
152
+
153
+ return {"success": 0, "errors": []}
154
+
26
155
  def exists(self, qhash):
27
156
  return self.datastore.safelist.get_if_exists(qhash, as_obj=False)
28
157
 
@@ -71,7 +71,6 @@ DOCKER_CONFIGURATION_PATH = os.getenv('DOCKER_CONFIGURATION_PATH', None)
71
71
  DOCKER_CONFIGURATION_VOLUME = os.getenv('DOCKER_CONFIGURATION_VOLUME', None)
72
72
 
73
73
  SERVICE_API_HOST = os.getenv('SERVICE_API_HOST', None)
74
- UI_SERVER = os.getenv('UI_SERVER', None)
75
74
  INTERNAL_ENCRYPT = bool(SERVICE_API_HOST and SERVICE_API_HOST.startswith('https'))
76
75
  SERVICE_PREFIX = 'alsvc-'
77
76
 
@@ -274,7 +273,7 @@ class ScalerServer(ThreadedCoreBase):
274
273
  core_env: dict[str, str] = {}
275
274
  # If we have privileged services, we must be able to pass the necessary environment variables for them to
276
275
  # function properly.
277
- for secret in re.findall(r'\${\w+}', open('/etc/assemblyline/config.yml', 'r').read()) + ['UI_SERVER']:
276
+ for secret in re.findall(r'\${\w+}', open('/etc/assemblyline/config.yml', 'r').read()):
278
277
  env_name = secret.strip("${}")
279
278
  try:
280
279
  core_env[env_name] = os.environ[env_name]
@@ -0,0 +1,210 @@
1
+ import logging
2
+
3
+ from assemblyline.common import forge
4
+ from assemblyline.common.isotime import iso_to_epoch, now_as_iso
5
+ from assemblyline.common.memory_zip import InMemoryZip
6
+ from assemblyline.datastore.helper import AssemblylineDatastore
7
+ from assemblyline.odm.messages.changes import Operation
8
+ from assemblyline.odm.models.service import SIGNATURE_DELIMITERS
9
+ from assemblyline.odm.models.signature import DEPLOYED_STATUSES, STALE_STATUSES, DRAFT_STATUSES
10
+
11
+
12
+ DEFAULT_DELIMITER = "\n\n"
13
+ CLASSIFICATION = forge.get_classification()
14
+
15
+
16
+ # Signature class
17
+ class SignatureClient:
18
+ """A helper class to simplify signature management for privileged services and service-server."""
19
+
20
+ def __init__(self, datastore: AssemblylineDatastore = None, config=None):
21
+ self.log = logging.getLogger('assemblyline.signature_client')
22
+ self.config = config or forge.CachedObject(forge.get_config)
23
+ self.datastore = datastore or forge.get_datastore(self.config)
24
+ self.service_list = forge.CachedObject(self.datastore.list_all_services, kwargs=dict(as_obj=False, full=True))
25
+ self.delimiters = forge.CachedObject(self._get_signature_delimiters)
26
+
27
+ def _get_signature_delimiters(self):
28
+ signature_delimiters = {}
29
+ for service in self.service_list:
30
+ if service.get("update_config", {}).get("generates_signatures", False):
31
+ signature_delimiters[service['name'].lower()] = self._get_signature_delimiter(service['update_config'])
32
+ return signature_delimiters
33
+
34
+ def _get_signature_delimiter(self, update_config):
35
+ delimiter_type = update_config['signature_delimiter']
36
+ if delimiter_type == 'custom':
37
+ delimiter = update_config['custom_delimiter'].encode().decode('unicode-escape')
38
+ else:
39
+ delimiter = SIGNATURE_DELIMITERS.get(delimiter_type, '\n\n')
40
+ return {'type': delimiter_type, 'delimiter': delimiter}
41
+
42
+ def add_update(self, data, dedup_name=True):
43
+ if data.get('type', None) is None or data['name'] is None or data['data'] is None:
44
+ raise ValueError("Signature id, name, type and data are mandatory fields.")
45
+
46
+ # Compute signature ID if missing
47
+ data['signature_id'] = data.get('signature_id', data['name'])
48
+
49
+ key = f"{data['type']}_{data['source']}_{data['signature_id']}"
50
+
51
+ # Test signature name
52
+ if dedup_name:
53
+ check_name_query = f"name:\"{data['name']}\" " \
54
+ f"AND type:\"{data['type']}\" " \
55
+ f"AND source:\"{data['source']}\" " \
56
+ f"AND NOT id:\"{key}\""
57
+ other = self.datastore.signature.search(check_name_query, fl='id', rows='0')
58
+ if other['total'] > 0:
59
+ raise ValueError("A signature with that name already exists")
60
+
61
+ old = self.datastore.signature.get(key, as_obj=False)
62
+ op = Operation.Modified if old else Operation.Added
63
+ if old:
64
+ if old['data'] == data['data']:
65
+ return True, key, None
66
+
67
+ # Ensure that the last state change, if any, was made by a user and not a system account.
68
+ user_modified_last_state = old['state_change_user'] not in ['update_service_account', None]
69
+
70
+ # If rule state is moving to an active state but was disabled by a user before:
71
+ # Keep original inactive state, a user changed the state for a reason
72
+ if user_modified_last_state and data['status'] == 'DEPLOYED' and data['status'] != old['status']:
73
+ data['status'] = old['status']
74
+
75
+ # Preserve last state change
76
+ data['state_change_date'] = old['state_change_date']
77
+ data['state_change_user'] = old['state_change_user']
78
+
79
+ # Preserve signature stats
80
+ data['stats'] = old['stats']
81
+
82
+ # Save the signature
83
+ success = self.datastore.signature.save(key, data)
84
+ return success, key, op
85
+
86
+ def add_update_many(self, source, sig_type, data, dedup_name=True):
87
+ if source is None or sig_type is None or not isinstance(data, list):
88
+ raise ValueError("Source, source type and data are mandatory fields.")
89
+
90
+ # Test signature names
91
+ names_map = {x['name']: f"{x['type']}_{x['source']}_{x.get('signature_id', x['name'])}" for x in data}
92
+
93
+ skip_list = []
94
+ if dedup_name:
95
+ for item in self.datastore.signature.stream_search(f"type: \"{sig_type}\" AND source:\"{source}\"",
96
+ fl="id,name", as_obj=False, item_buffer_size=1000):
97
+ lookup_id = names_map.get(item['name'], None)
98
+ if lookup_id and lookup_id != item['id']:
99
+ skip_list.append(lookup_id)
100
+
101
+ if skip_list:
102
+ data = [
103
+ x for x in data
104
+ if f"{x['type']}_{x['source']}_{x.get('signature_id', x['name'])}" not in skip_list]
105
+
106
+ old_data = self.datastore.signature.multiget(list(names_map.values()), as_dictionary=True, as_obj=False,
107
+ error_on_missing=False)
108
+
109
+ plan = self.datastore.signature.get_bulk_plan()
110
+ for rule in data:
111
+ key = f"{rule['type']}_{rule['source']}_{rule.get('signature_id', rule['name'])}"
112
+ if key in old_data:
113
+ # Ensure that the last state change, if any, was made by a user and not a system account.
114
+ user_modified_last_state = old_data[key]['state_change_user'] not in ['update_service_account', None]
115
+
116
+ # If rule state is moving to an active state but was disabled by a user before:
117
+ # Keep original inactive state, a user changed the state for a reason
118
+ if user_modified_last_state and rule['status'] == 'DEPLOYED' and rule['status'] != old_data[key][
119
+ 'status']:
120
+ rule['status'] = old_data[key]['status']
121
+
122
+ # Preserve last state change
123
+ rule['state_change_date'] = old_data[key]['state_change_date']
124
+ rule['state_change_user'] = old_data[key]['state_change_user']
125
+
126
+ # Preserve signature stats
127
+ rule['stats'] = old_data[key]['stats']
128
+
129
+ plan.add_upsert_operation(key, rule)
130
+
131
+ if not plan.empty:
132
+ res = self.datastore.signature.bulk(plan)
133
+ return {"success": len(res['items']), "errors": res['errors'], "skipped": skip_list}
134
+
135
+ return {"success": 0, "errors": [], "skipped": skip_list}
136
+
137
+ def change_status(self, signature_id, status, user={}):
138
+ possible_statuses = DEPLOYED_STATUSES + DRAFT_STATUSES
139
+ if status not in possible_statuses:
140
+ raise ValueError(f"You cannot apply the status {status} on yara rules.")
141
+
142
+ data = self.datastore.signature.get(signature_id, as_obj=False)
143
+ if data:
144
+ if user and not CLASSIFICATION.is_accessible(user['classification'],
145
+ data.get('classification', CLASSIFICATION.UNRESTRICTED)):
146
+ raise PermissionError("You are not allowed change status on this signature")
147
+
148
+ if data['status'] in STALE_STATUSES and status not in DRAFT_STATUSES:
149
+ raise ValueError(f"Only action available while signature in {data['status']} "
150
+ f"status is to change signature to a DRAFT status. ({', '.join(DRAFT_STATUSES)})")
151
+
152
+ if data['status'] in DEPLOYED_STATUSES and status in DRAFT_STATUSES:
153
+ raise ValueError(f"You cannot change the status of signature {signature_id} from "
154
+ f"{data['status']} to {status}.")
155
+
156
+ today = now_as_iso()
157
+ uname = user.get('uname')
158
+
159
+ if status not in ['DISABLED', 'INVALID', 'TESTING']:
160
+ query = f"status:{status} AND signature_id:{data['signature_id']} AND NOT id:{signature_id}"
161
+ others_operations = [
162
+ ('SET', 'last_modified', today),
163
+ ('SET', 'state_change_date', today),
164
+ ('SET', 'state_change_user', uname),
165
+ ('SET', 'status', 'DISABLED')
166
+ ]
167
+ self.datastore.signature.update_by_query(query, others_operations)
168
+
169
+ operations = [
170
+ ('SET', 'last_modified', today),
171
+ ('SET', 'state_change_date', today),
172
+ ('SET', 'state_change_user', uname),
173
+ ('SET', 'status', status)
174
+ ]
175
+
176
+ return self.datastore.signature.update(signature_id, operations), data
177
+ raise FileNotFoundError(f"Signature not found. ({signature_id})")
178
+
179
+ def download(self, query=None, access=None) -> bytes:
180
+ if not query:
181
+ query = "*"
182
+
183
+ output_files = {}
184
+
185
+ signature_list = sorted(
186
+ self.datastore.signature.stream_search(
187
+ query, fl="signature_id,type,source,data,order", access_control=access, as_obj=False,
188
+ item_buffer_size=1000),
189
+ key=lambda x: x['order'])
190
+
191
+ for sig in signature_list:
192
+ out_fname = f"{sig['type']}/{sig['source']}"
193
+ if self.delimiters.get(sig['type'], {}).get('type', None) == 'file':
194
+ out_fname = f"{out_fname}/{sig['signature_id']}"
195
+ output_files.setdefault(out_fname, [])
196
+ output_files[out_fname].append(sig['data'])
197
+
198
+ output_zip = InMemoryZip()
199
+ for fname, data in output_files.items():
200
+ separator = self.delimiters.get(fname.split('/')[0], {}).get('delimiter', DEFAULT_DELIMITER)
201
+ output_zip.append(fname, separator.join(data))
202
+
203
+ return output_zip.read()
204
+
205
+ def update_available(self, since='', sig_type='*'):
206
+ since = since or '1970-01-01T00:00:00.000000Z'
207
+ last_update = iso_to_epoch(since)
208
+ last_modified = iso_to_epoch(self.datastore.get_signature_last_modified(sig_type))
209
+
210
+ return last_modified > last_update
@@ -39,14 +39,13 @@ NAMESPACE = os.getenv('NAMESPACE', None)
39
39
  INHERITED_VARIABLES: list[str] = ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'] + \
40
40
  [
41
41
  secret.strip("${}")
42
- for secret in re.findall(r'\${\w+}', open('/etc/assemblyline/config.yml', 'r').read()) + ['UI_SERVER']]
42
+ for secret in re.findall(r'\${\w+}', open('/etc/assemblyline/config.yml', 'r').read())]
43
43
 
44
44
  CONFIGURATION_HOST_PATH = os.getenv('CONFIGURATION_HOST_PATH', 'service_config')
45
45
  CONFIGURATION_CONFIGMAP = os.getenv('KUBERNETES_AL_CONFIG', None)
46
46
  AL_CORE_NETWORK = os.environ.get("AL_CORE_NETWORK", 'core')
47
47
 
48
48
  SERVICE_API_HOST = os.getenv('SERVICE_API_HOST')
49
- UI_SERVER = os.getenv('UI_SERVER')
50
49
  RELEASE_NAME = os.getenv('RELEASE_NAME')
51
50
 
52
51
 
@@ -1,13 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.dev5
3
+ Version: 4.5.0.dev7
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
7
7
  Author-email: assemblyline@cyber.gc.ca
8
8
  License: MIT
9
9
  Keywords: assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs
10
- Platform: UNKNOWN
11
10
  Classifier: Development Status :: 5 - Production/Stable
12
11
  Classifier: Intended Audience :: Developers
13
12
  Classifier: Topic :: Software Development :: Libraries
@@ -59,5 +58,3 @@ Make sure the different services get their latest update files.
59
58
  ##### Workflow
60
59
 
61
60
  Run the different workflows in the system and apply their labels, priority and status.
62
-
63
-
@@ -7,6 +7,7 @@ assemblyline_core/__init__.py
7
7
  assemblyline_core/badlist_client.py
8
8
  assemblyline_core/safelist_client.py
9
9
  assemblyline_core/server_base.py
10
+ assemblyline_core/signature_client.py
10
11
  assemblyline_core/submission_client.py
11
12
  assemblyline_core/tasking_client.py
12
13
  assemblyline_core.egg-info/PKG-INFO
@@ -70,12 +71,15 @@ assemblyline_core/vacuum/worker.py
70
71
  assemblyline_core/workflow/__init__.py
71
72
  assemblyline_core/workflow/run_workflow.py
72
73
  test/test_alerter.py
74
+ test/test_badlist_client.py
73
75
  test/test_dispatcher.py
74
76
  test/test_expiry.py
75
77
  test/test_plumber.py
76
78
  test/test_replay.py
79
+ test/test_safelist_client.py
77
80
  test/test_scaler.py
78
81
  test/test_scheduler.py
82
+ test/test_signature_client.py
79
83
  test/test_simulation.py
80
84
  test/test_vacuum.py
81
85
  test/test_worker_ingest.py