assemblyline-core 4.6.1.dev44__tar.gz → 4.6.1.dev49__tar.gz

{assemblyline_core-4.6.1.dev44 → assemblyline_core-4.6.1.dev49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.1.dev44
+Version: 4.6.1.dev49
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.6.1.dev49/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.6.1.dev49

{assemblyline_core-4.6.1.dev44 → assemblyline_core-4.6.1.dev49}/assemblyline_core/plumber/run_plumber.py

@@ -200,55 +200,43 @@ class Plumber(CoreBase):
         self.log.info(f"Done watching {service_name} service queue")
 
     def user_apikey_cleanup(self):
-        query = "id:*"
-        offset = 0
-        rows = 100
-        total = 1
-        cur_total = 0
+        expiry_ts = None
+        if self.config.auth.apikey_max_dtl is not None:
+            expiry_ts = now_as_iso(self.config.auth.apikey_max_dtl * DAY_IN_SECONDS)
 
-        config = get_config()
-        apikey_max_dtl = config.auth.apikey_max_dtl
+        for user in self.datastore.user.stream_search(query="*", fl="*", as_obj=False):
+            uname = user['uname']
+            apikeys = user['apikeys']
 
-        expiry_ts = now_as_iso(apikey_max_dtl * DAY_IN_SECONDS) if apikey_max_dtl is not None else None
+            for key in apikeys:
+                old_apikey = apikeys[key]
+                key_id = get_apikey_id(key, uname)
 
-        while cur_total < total:
-            result = self.datastore.user.search(query, offset=offset, rows=rows)
-            total = result.get('total', 0)
-            cur_total = cur_total + (result.get("count", total))
+                roles = None
+                if old_apikey['acl'] == ["C"]:
 
-            # check for API keys in total
-            users = result.get('items', [])
-
-            for u in users:
-                uname = u['uname']
-                user = self.datastore.user.get(uname)
-                apikeys = user.apikeys
-
-                for key in apikeys:
-                    old_apikey = apikeys[key]
-                    key_id = get_apikey_id(key, uname)
-
-                    roles = None
-                    if old_apikey['acl'] == ["C"]:
+                    roles = [r for r in old_apikey['roles']
+                                if r in load_roles(user['type'], user['roles'])]
 
-                        roles = [r for r in old_apikey['roles']
-                                    if r in load_roles(user['type'], user['roles'])]
+                else:
+                    roles = [r for r in load_roles_form_acls(old_apikey['acl'], roles)
+                            if r in load_roles(user['type'], user['roles'])]
+                new_apikey = {
+                    "password": old_apikey['password'],
+                    "acl": old_apikey['acl'],
+                    "uname": uname,
+                    "key_name": key,
+                    "roles": roles,
+                    "expiry_ts": expiry_ts
+                }
+                self.datastore.apikey.save(key_id, new_apikey)
 
-                    else:
-                        roles = [r for r in load_roles_form_acls(old_apikey['acl'], roles)
-                                if r in load_roles(user['type'], user['roles'])]
-                    new_apikey = {
-                        "password": old_apikey['password'],
-                        "acl": old_apikey['acl'],
-                        "uname": uname,
-                        "key_name": key,
-                        "roles": roles,
-                        "expiry_ts": expiry_ts
-                    }
-                    self.datastore.apikey.save(key_id, new_apikey)
+            user['apikeys'] = {}
+            self.datastore.user.save(uname, user)
 
-                user['apikeys'] = {}
-                self.datastore.user.save(uname, user)
+        # Commit changes made to indices
+        self.datastore.user.commit()
+        self.datastore.apikey.commit()
 
     def migrate_user_settings(self):
         service_list = self.datastore.list_all_services(as_obj=False)

{assemblyline_core-4.6.1.dev44 → assemblyline_core-4.6.1.dev49}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.1.dev44
+Version: 4.6.1.dev49
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline_core-4.6.1.dev44 → assemblyline_core-4.6.1.dev49}/test/test_plumber.py

@@ -8,6 +8,7 @@ from redis import Redis
 
 from assemblyline.odm.messages.task import Task
 from assemblyline.odm.models.service import Service
+from assemblyline.odm.models.user import ApiKey, User
 from assemblyline.odm.random_data import random_model_obj
 
 
@@ -120,27 +121,38 @@ def test_user_setting_migrations(datastore_connection):
 
     # Create a user with old settings (format prior to 4.6)
     settings = {'classification': 'TLP:CLEAR', 'deep_scan': False, 'description': '', 'download_encoding': 'cart', 'default_external_sources': ['Malware Bazaar', 'VirusTotal'], 'default_zip_password': 'zippy', 'executive_summary': False, 'expand_min_score': 500, 'generate_alert': False, 'ignore_cache': False, 'ignore_dynamic_recursion_prevention': False, 'ignore_recursion_prevention': False, 'ignore_filtering': False, 'malicious': False, 'priority': 369, 'profile': False, 'service_spec': {'AVClass': {'include_malpedia_dataset': False}}, 'services': {'selected': ['Extraction', 'ConfigExtractor', 'YARA'], 'excluded': [], 'rescan': [], 'resubmit': [], 'runtime_excluded': []}, 'submission_view': 'report', 'ttl': 0}
+
+    user_account = random_model_obj(User, as_json=True)
+    user_account['uname'] = "admin"
+    user_account['apikeys'] = {'test': random_model_obj(ApiKey, as_json=True)}
     datastore_connection.ds.client.index(index="user_settings", id="admin", document=settings)
+    datastore_connection.ds.client.index(index="user", id="admin", document=user_account)
+
     datastore_connection.user_settings.commit()
+    datastore_connection.user.commit()
 
     # Initiate the migration
+    plumber.user_apikey_cleanup()
     plumber.migrate_user_settings()
 
     # Check that the settings have been migrated
     migrated_settings = datastore_connection.user_settings.get("admin", as_obj=False)
 
+    # Check to see if API keys for the user were transferred to the new index
+    assert datastore_connection.apikey.search('uname:admin', rows=0)['total'] > 0
+
     # Deprecated settings should be removed
     assert "ignore_dynamic_recursion_prevention" not in migrated_settings
 
     # All former submission settings at the root-level should be moved to submission profiles
     assert all([key not in migrated_settings for key in SubmissionProfileParams.fields().keys()] )
 
-    for name, settings in migrated_settings['submission_profiles'].items():
+    for settings in migrated_settings['submission_profiles'].values():
         assert settings['classification'] == 'TLP:C'
         assert settings['deep_scan'] is False
         assert settings['generate_alert'] is False
         assert settings['ignore_cache'] is False
         assert settings['priority'] == 369
-        # Full service spec should be preserved in default profile, but not in pre-defined ones
-        assert settings['service_spec'] == {} if name != "default" else {'AVClass': {'include_malpedia_dataset': False}}
+        # Full service spec should be preserved in default profile (along with others by default if there's no restricted parameters)
+        assert settings['service_spec'] == {'AVClass': {'include_malpedia_dataset': False}}
         assert settings['ttl'] == 0

assemblyline_core-4.6.1.dev44/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.6.1.dev44