assemblyline-core 4.5.1.dev368__tar.gz → 4.5.1.dev369__tar.gz

{assemblyline-core-4.5.1.dev368 → assemblyline-core-4.5.1.dev369}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev368
+Version: 4.5.1.dev369
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev369/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev369

assemblyline-core-4.5.1.dev369/assemblyline_core/__init__.py

@@ -0,0 +1,10 @@
+PAUSABLE_COMPONENTS = ['ingester', 'dispatcher']
+
+def normalize_hashlist_item(tag_type: str, tag_value: str) -> str:
+    # Normalize tag data pertaining to domains or URIs
+    if tag_type.endswith('.domain'):
+        tag_value = tag_value.lower()
+    elif tag_type.endswith('.uri'):
+        hostname = tag_value.split('//', 1)[1].split('/', 1)[0]
+        tag_value = tag_value.replace(hostname, hostname.lower(), 1)
+    return tag_value

{assemblyline-core-4.5.1.dev368 → assemblyline-core-4.5.1.dev369}/assemblyline_core/badlist_client.py

@@ -8,6 +8,8 @@ from assemblyline.datastore.helper import AssemblylineDatastore
 from assemblyline.odm.models.user import ROLES
 from assemblyline.remote.datatypes.lock import Lock
 
+from assemblyline_core import normalize_hashlist_item
+
 CHUNK_SIZE = 1000
 CLASSIFICATION = forge.get_classification()
 
@@ -43,6 +45,9 @@ class BadlistClient:
             if tag_data is None or 'type' not in tag_data or 'value' not in tag_data:
                 raise ValueError("Tag data not found")
 
+            # Normalize tag data before further processing
+            tag_data['value'] = normalize_hashlist_item(tag_data['type'], tag_data['value'])
+
             hashed_value = f"{tag_data['type']}: {tag_data['value']}".encode('utf8')
             data['hashes'] = {
                 'sha256': hashlib.sha256(hashed_value).hexdigest()
@@ -143,7 +148,7 @@ class BadlistClient:
         lookup_keys = []
         for tag_type, tag_values in tag_map.items():
             for tag_value in tag_values:
-                lookup_keys.append(hashlib.sha256(f"{tag_type}: {tag_value}".encode('utf8')).hexdigest())
+                lookup_keys.append(hashlib.sha256(f"{tag_type}: {normalize_hashlist_item(tag_type, tag_value)}".encode('utf8')).hexdigest())
 
         # Elasticsearch's result window can't be more than 10000 rows
         # we will query for matches in chunks

{assemblyline-core-4.5.1.dev368 → assemblyline-core-4.5.1.dev369}/assemblyline_core/safelist_client.py

@@ -8,6 +8,8 @@ from assemblyline.datastore.helper import AssemblylineDatastore
 from assemblyline.odm.models.user import ROLES
 from assemblyline.remote.datatypes.lock import Lock
 
+from assemblyline_core import normalize_hashlist_item
+
 CLASSIFICATION = forge.get_classification()
 
 
@@ -43,6 +45,9 @@ class SafelistClient:
             if tag_data is None or 'type' not in tag_data or 'value' not in tag_data:
                 raise ValueError("Tag data not found")
 
+            # Normalize tag data before further processing
+            tag_data['value'] = normalize_hashlist_item(tag_data['type'], tag_data['value'])
+
             hashed_value = f"{tag_data['type']}: {tag_data['value']}".encode('utf8')
             data['hashes'] = {
                 'sha256': hashlib.sha256(hashed_value).hexdigest()

{assemblyline-core-4.5.1.dev368 → assemblyline-core-4.5.1.dev369}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev368
+Version: 4.5.1.dev369
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.1.dev368 → assemblyline-core-4.5.1.dev369}/test/test_badlist_client.py

@@ -1,12 +1,18 @@
 
 import hashlib
 import random
+import time
 from copy import deepcopy
-import pytest
 
+import pytest
 from assemblyline.common.forge import get_classification
 from assemblyline.common.isotime import iso_to_epoch
-from assemblyline.odm.random_data import create_users, create_badlists, wipe_users, wipe_badlist
+from assemblyline.odm.random_data import (
+    create_badlists,
+    create_users,
+    wipe_badlist,
+    wipe_users,
+)
 from assemblyline.odm.randomizer import get_random_hash
 from assemblyline_core.badlist_client import BadlistClient, InvalidBadhash
 
@@ -288,3 +294,39 @@ def test_badlist_update_conflict(client):
         client.add_update(sl_data)
 
     assert 'has a type conflict:' in conflict_exc.value.args[0]
+
+def test_badlist_tag_normalization(client):
+    tag_type = 'network.static.uri'
+    tag_value = 'https://BaD.com/About'
+
+    normalized_value = 'https://bad.com/About'
+    hashed_value = f"{tag_type}: {normalized_value}".encode('utf8')
+    expected_qhash = hashlib.sha256(hashed_value).hexdigest()
+
+    # Generate a random badlist
+    sl_data = {
+        'attribution': {
+            'actor': ["SOMEONE!"],
+            'campaign': None,
+            'category': None,
+            'exploit': None,
+            'implant': None,
+            'family': None,
+            'network': None
+        },
+        'dtl': 15,
+        'tag': {'type': tag_type,
+                'value': tag_value},
+        'sources': [BAD_SOURCE, ADMIN_SOURCE],
+        'type': 'tag'
+    }
+
+    client.add_update(sl_data)
+
+    # Assert that item got created with the expected ID from the normalized tag value
+    assert client.datastore.badlist.exists(expected_qhash)
+    time.sleep(1)
+
+    # Assert that the tag exists in either format (within reason)
+    assert client.exists_tags({tag_type: [tag_value]})
+    assert client.exists_tags({tag_type: [normalized_value]})

{assemblyline-core-4.5.1.dev368 → assemblyline-core-4.5.1.dev369}/test/test_safelist_client.py

@@ -1,15 +1,20 @@
 
 import hashlib
 import random
+import time
 from copy import deepcopy
 
 import pytest
-
 from assemblyline.common.forge import get_classification
 from assemblyline.common.isotime import iso_to_epoch, now_as_iso
-from assemblyline.odm.random_data import create_users, create_safelists, wipe_users, wipe_safelist
+from assemblyline.odm.random_data import (
+    create_safelists,
+    create_users,
+    wipe_safelist,
+    wipe_users,
+)
 from assemblyline.odm.randomizer import get_random_hash
-from assemblyline_core.safelist_client import SafelistClient, InvalidSafehash
+from assemblyline_core.safelist_client import InvalidSafehash, SafelistClient
 
 add_hash_file = "10" + get_random_hash(62)
 add_error_hash = "11" + get_random_hash(62)
@@ -326,3 +331,38 @@ def test_safelist_update_conflict(client):
         client.add_update(sl_data)
 
     assert 'has a type conflict:' in conflict_exc.value.args[0]
+
+def test_safelist_tag_normalization(client):
+    tag_type = 'network.static.uri'
+    tag_value = 'https://gOOd.com/About'
+
+    normalized_value = 'https://good.com/About'
+    hashed_value = f"{tag_type}: {normalized_value}".encode('utf8')
+    expected_qhash = hashlib.sha256(hashed_value).hexdigest()
+
+    # Generate a safelist item
+    sl_data = {
+        'attribution': {
+            'actor': ["SOMEONE!"],
+            'campaign': None,
+            'category': None,
+            'exploit': None,
+            'implant': None,
+            'family': None,
+            'network': None
+        },
+        'dtl': 15,
+        'tag': {'type': tag_type,
+                'value': tag_value},
+        'sources': [NSRL_SOURCE, ADMIN_SOURCE],
+        'type': 'tag'
+    }
+
+    client.add_update(sl_data)
+
+    # Assert that item got created with the expected ID from the normalized tag value
+    assert client.datastore.safelist.exists(expected_qhash)
+    time.sleep(1)
+
+    # Assert that the tag exists in either format (within reason)
+    assert client.get_safelisted_tags([tag_type])['match'][tag_type] == [normalized_value]

assemblyline-core-4.5.1.dev368/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev368

assemblyline-core-4.5.1.dev368/assemblyline_core/__init__.py

@@ -1 +0,0 @@
-PAUSABLE_COMPONENTS = ['ingester', 'dispatcher']