assemblyline-core 4.5.1.dev165__tar.gz → 4.5.1.dev167__tar.gz

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev165
+Version: 4.5.1.dev167
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev167/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev167

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/assemblyline_core/badlist_client.py

@@ -52,9 +52,9 @@ class BadlistClient:
             data.setdefault('file', {})
 
         # Ensure expiry_ts is set on tag-related items
-        dtl = data.pop('dtl', None) or self.config.core.expiry.badlisted_tag_dtl * 24 * 3600
+        dtl = data.pop('dtl', None) or self.config.core.expiry.badlisted_tag_dtl
         if dtl:
-            data['expiry_ts'] = now_as_iso(dtl)
+            data['expiry_ts'] = now_as_iso(dtl * 24 * 3600)
 
         # Set last updated
         data['added'] = data['updated'] = now_as_iso()

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/assemblyline_core/ingester/ingester.py

@@ -595,7 +595,6 @@ class Ingester(ThreadedCoreBase):
                 task.params.submitter) if g in str(task.params.classification)]
 
         # Check if this file is already being processed
-        self.stamp_filescore_key(task)
         pprevious, previous, score = None, None, None
         if not param.ignore_cache:
             pprevious, previous, score, _ = self.check(task, count_miss=False)
@@ -627,6 +626,33 @@ class Ingester(ThreadedCoreBase):
         # (So we don't end up dropping the resubmission).
         if previous:
             self.counter.increment('duplicates')
+
+            if self.config.core.ingester.always_create_submission:
+                # Create a submission record based on the cache hit
+                submission = self.datastore.submission.get(previous, as_obj=False)
+
+                # Assign the current submission as the PSID for the new submission
+                pprevious = previous
+                previous = task.ingest_id
+                task.params.psid = pprevious
+
+                submission.update({
+                    'archived': False,
+                    'archive_ts': None,
+                    'classification': task.params.classification,
+                    'expiry_ts':now_as_iso(task.params.ttl * 24 * 60 * 60),
+                    'from_archive': False,
+                    'metadata': task.submission.metadata,
+                    'params': task.params.as_primitives(),
+                    'sid': previous,
+                    'to_be_deleted': False,
+                    'times': {
+                        'submitted': task.ingest_time,
+                        'completed': "NOW"
+                    },
+                })
+                self.datastore.submission.save(previous, submission)
+
             self.finalize(pprevious, previous, score, task, cache=True)
 
             # On cache hits of any kind we want to send out a completed message

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/assemblyline_core/safelist_client.py

@@ -69,9 +69,9 @@ class SafelistClient:
             data.setdefault('file', {})
 
         # Ensure expiry_ts is set on tag-related items
-        dtl = data.pop('dtl', None) or self.config.core.expiry.safelisted_tag_dtl * 24 * 3600
+        dtl = data.pop('dtl', None) or self.config.core.expiry.safelisted_tag_dtl
         if dtl:
-            data['expiry_ts'] = now_as_iso(dtl)
+            data['expiry_ts'] = now_as_iso(dtl * 24 * 3600)
 
         # Set last updated
         data['added'] = data['updated'] = now_as_iso()

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -27,7 +27,6 @@ from kubernetes.client import V1Deployment, V1DeploymentSpec, V1PodTemplateSpec,
     V1NetworkPolicyIngressRule, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Service, \
     V1ServiceSpec, V1ServicePort, V1PodSecurityContext, V1Probe, V1ExecAction, V1SecurityContext, \
     V1Affinity, V1NodeAffinity, V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, CoreV1Event, V1Toleration
-
 from kubernetes.client.rest import ApiException
 from assemblyline.odm.models.service import DependencyConfig, DockerConfig, PersistentVolume
 

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/assemblyline_core/scaler/scaler_server.py

@@ -17,6 +17,7 @@ import copy
 from contextlib import contextmanager
 
 import elasticapm
+import json
 import yaml
 
 from assemblyline.remote.datatypes.queues.named import NamedQueue
@@ -364,7 +365,8 @@ class ScalerServer(ThreadedCoreBase):
                 self.controller.core_mounts.append((DOCKER_CONFIGURATION_VOLUME, '/etc/assemblyline/'))
 
                 with open(os.path.join(DOCKER_CONFIGURATION_PATH, 'config.yml'), 'w') as handle:
-                    yaml.dump(self.config.as_primitives(), handle)
+                    # Convert to JSON before converting to YAML to account for direct ODM representation errors
+                    yaml.dump(json.loads(self.config.json()), handle)
 
                 with open(os.path.join(DOCKER_CONFIGURATION_PATH, 'classification.yml'), 'w') as handle:
                     yaml.dump(get_classification().original_definition, handle)

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev165
+Version: 4.5.1.dev167
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.1.dev165 → assemblyline-core-4.5.1.dev167}/test/test_worker_ingest.py

@@ -193,3 +193,56 @@ def test_ingest_size_error(ingest_harness):
     queue = ingester.notification_queues[queue_name]
     message = queue.pop()
     assert message is not None
+
+def test_ingest_always_create_submission(ingest_harness):
+    datastore, ingester, in_queue = ingest_harness
+
+    # Simulate configuration where we'll always create a submission
+    ingester.config.core.ingester.always_create_submission = True
+    get_if_exists = datastore.filescore.get_if_exists
+    try:
+        # Add a valid file score for all files
+        from assemblyline.odm.models.filescore import FileScore
+        from assemblyline.odm.models.submission import Submission
+        datastore.filescore.get_if_exists = mock.MagicMock(
+            return_value=FileScore(dict(psid='000', expiry_ts=0, errors=0, score=10, sid='001', time=time.time()))
+        )
+        # Create a submission for cache hit
+        old_sub = random_minimal_obj(Submission)
+        old_sub.sid = '001'
+        old_sub.params.psid = '000'
+        old_sub = old_sub.as_primitives()
+        datastore.submission.save('001', old_sub)
+
+        # Ingest a file
+        submission_msg = make_message(message={'sid': '002', 'metadata': {'blah': 'blah'}})
+        submission_msg['sid'] = '002'
+        in_queue.push(submission_msg)
+        ingester.handle_ingest()
+
+        # No file has made it into the internal buffer => cache hit and drop
+        datastore.filescore.get_if_exists.assert_called_once()
+        ingester.counter.increment.assert_any_call('cache_hit')
+        ingester.counter.increment.assert_any_call('duplicates')
+        assert ingester.unique_queue.length() == 0
+        assert ingester.ingest_queue.length() == 0
+
+        # Check to see if new submission was created
+        new_sub = datastore.submission.get_if_exists('002', as_obj=False)
+        assert new_sub and new_sub['params']['psid'] == old_sub['sid']
+
+        # Check to see if certain properties are same (anything relating to analysis)
+        assert all([old_sub.get(attr) == new_sub.get(attr) \
+                    for attr in ['error_count', 'errors', 'file_count', 'files', 'max_score', 'results', 'state', 'verdict']])
+
+        # Check to see if certain properties are different
+        # (anything that isn't related to analysis but can be set at submission time)
+        assert all([old_sub.get(attr) != new_sub.get(attr) \
+                    for attr in ['expiry_ts', 'metadata', 'params', 'times']])
+
+        # Check to see if certain properties have been nullified
+        # (properties that are set outside of submission)
+        assert not all([new_sub.get(attr) \
+                        for attr in ['archived', 'archive_ts', 'to_be_deleted', 'from_archive']])
+    finally:
+        datastore.filescore.get_if_exists = get_if_exists

assemblyline-core-4.5.1.dev165/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev165