assemblyline 4.7.5.dev77__tar.gz → 4.7.5.dev79__tar.gz

{assemblyline-4.7.5.dev77/assemblyline.egg-info → assemblyline-4.7.5.dev79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev77
+Version: 4.7.5.dev79
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.5.dev79/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.5.dev79

{assemblyline-4.7.5.dev77 → assemblyline-4.7.5.dev79}/assemblyline/datastore/helper.py

@@ -374,7 +374,7 @@ class AssemblylineDatastore(object):
                         new_file_class = cl_engine.min_classification(new_file_class, c)
 
                     # Find the results for that classification and alter them if the new classification does not match
-                    for item in self.result.stream_search(f"id:{f}*", fl="classification,id,_index", as_obj=False):
+                    for item in self.result.stream_search(f"sha256:{f}", fl="classification,id,_index", as_obj=False):
                         try:
                             new_class = cl_engine.max_classification(
                                 item.get('classification', cl_engine.UNRESTRICTED), new_file_class)
@@ -404,119 +404,6 @@ class AssemblylineDatastore(object):
         # Proceed with plan
         self.multi_index_bulk([s_plan, st_plan, ss_plan, e_plan, er_plan, r_plan, f_plan, fscore_plan])
 
-    @elasticapm.capture_span(span_type='datastore')
-    def delete_submission_tree(
-            self, sid: str, cl_engine: Classification = forge.get_classification(),
-            cleanup: bool = True, transport: FileStore | None = None) -> None:
-        submission = self.submission.get(sid, as_obj=False)
-        if not submission:
-            return
-
-        # Gather file list
-        errors = submission['errors']
-        results = submission["results"]
-        files_to_delete = set()
-        fix_classification_files = set()
-        supp_map: dict[str, set[str]] = {}
-
-        temp_files = {x[:64] for x in errors}
-        temp_files.update(x[:64] for x in results)
-
-        # Gather all supplementary files and extracted files
-        for result in self.result.search("response.supplementary.sha256:* OR response.extracted.sha256:*",
-                                         fl="sha256,response.supplementary.sha256,response.extracted.sha256",
-                                         rows=len(results), as_obj=False, key_space=results)['items']:
-            sha256 = result['sha256']
-            if sha256 not in supp_map:
-                supp_map[sha256] = set()
-            for supp in result['response'].get('supplementary', []):
-                supp_map[sha256].add(supp['sha256'])
-            for extracted in result['response'].get('extracted', []):
-                temp_files.add(extracted['sha256'])
-
-        # Inspect each files to see if they are reused
-        for temp in temp_files:
-            # Check if we delete the file or update the classification
-            if self.submission.search(f"NOT sid:{sid} AND (errors:{temp}* OR results:{temp}*)", rows=0, as_obj=False)["total"] == 0:
-                files_to_delete.add(temp)
-                files_to_delete |= supp_map.pop(temp, set())
-            else:
-                fix_classification_files.add(temp)
-
-        # Filter results and errors
-        errors = [x for x in errors if x[:64] in files_to_delete]
-        results = [x for x in results if x[:64] in files_to_delete]
-
-        # Delete files, errors, results that were only used once
-        for e in errors:
-            self.error.delete(e)
-        for r in results:
-            if r.endswith(".e"):
-                self.emptyresult.delete(r)
-            else:
-                self.result.delete(r)
-        for f in files_to_delete:
-            self.file.delete(f)
-            if transport:
-                transport.delete(f)
-
-        if fix_classification_files and cleanup:
-            # Fix classification for the files that remain in the system
-            for f in fix_classification_files:
-                cur_file = self.file.get(f, as_obj=False)
-                if cur_file:
-                    # Find possible classification for the file in the system
-                    query = f"NOT id:{sid} AND (files.sha256:{f} OR results:{f}* OR errors:{f}*)"
-                    classifications = list(self.submission.facet('classification', query=query, size=100).keys())
-
-                    if len(classifications) > 0:
-                        new_file_class = classifications[0]
-                    else:
-                        new_file_class = cl_engine.UNRESTRICTED
-
-                    for c in classifications:
-                        new_file_class = cl_engine.min_classification(new_file_class, c)
-
-                    # Find the results for that classification and alter them if the new classification does not match
-                    for item in self.result.stream_search(f"id:{f}*", fl="classification,id", as_obj=False):
-                        try:
-                            new_class = cl_engine.max_classification(
-                                item.get('classification', cl_engine.UNRESTRICTED), new_file_class)
-                        except InvalidClassification:
-                            # If we can't find common grounds for reclassifying the results... we simply wont.
-                            continue
-
-                        if item.get('classification', cl_engine.UNRESTRICTED) != new_class:
-                            parts = cl_engine.get_access_control_parts(new_class)
-                            update_params = [(ESCollection.UPDATE_SET, 'classification', new_class)]
-                            update_params.extend([(ESCollection.UPDATE_SET, k, v) for k, v in parts.items()])
-                            self.result.update(item['id'], update_params)
-
-                    # Alter the file classification if the new classification does not match
-                    if cur_file['classification'] != new_file_class:
-                        parts = cl_engine.get_access_control_parts(new_file_class)
-                        update_params = [(ESCollection.UPDATE_SET, 'classification', new_file_class)]
-                        update_params.extend([(ESCollection.UPDATE_SET, k, v) for k, v in parts.items()])
-                        self.file.update(f, update_params)
-
-                    # Fix associated supplementary files
-                    for supp in supp_map.get(f, set()):
-                        cur_supp = self.file.get(supp, as_obj=False)
-                        if cur_supp:
-                            if cur_supp['classification'] != new_file_class:
-                                parts = cl_engine.get_access_control_parts(new_file_class)
-                                update_params = [(ESCollection.UPDATE_SET, 'classification', new_file_class)]
-                                update_params.extend([(ESCollection.UPDATE_SET, k, v) for k, v in parts.items()])
-                                self.file.update(supp, update_params)
-
-        # Delete the submission and cached trees and summaries
-        # NOTE: we will use the search operation because there should not be a lot of submission_tree or summary
-        self.submission.delete(sid)
-        for t in self.submission_tree.search(f"id:{sid}*", fl="id", rows=1000, as_obj=False)['items']:
-            self.submission_tree.delete(t['id'])
-        for s in self.submission_summary.search(f"id:{sid}*", fl="id", rows=1000, as_obj=False)['items']:
-            self.submission_summary.delete(s['id'])
-
     @elasticapm.capture_span(span_type='datastore')
     def get_all_heuristics(self):
         return {h['id']: h for h in self.heuristic.stream_search("id:*", as_obj=False)}
@@ -1317,192 +1204,6 @@ class AssemblylineDatastore(object):
             except VersionConflictException as vce:
                 log.info(f"Retrying save or freshen due to version conflict: {str(vce)}")
 
-    def _get_verdict_from_score(self, score):
-        if score >= config.submission.verdicts.malicious:
-            return "malicious"
-        elif score >= config.submission.verdicts.highly_suspicious:
-            return "highly suspicious"
-        elif score >= config.submission.verdicts.suspicious:
-            return "suspicious"
-        elif score >= config.submission.verdicts.info:
-            return "informative"
-        else:
-            return "safe"
-
-    @elasticapm.capture_span(span_type='datastore')
-    def _fix_section_data(self, result, min_score):
-        new_sections = []
-        for section in result['result'].get('sections', []):
-            # Skip section with a small score
-            if section.get('heuristic', {}).get('score', -1) < min_score:
-                continue
-
-            # Rewrite heuristic score as a verdict
-            if 'heuristic' in section:
-                heur_score = section['heuristic'].pop('score', 0)
-                section['heuristic']['verdict'] = self._get_verdict_from_score(heur_score)
-
-            # Loading JSON formatted sections
-            body_format = section['body_format']
-            section_body = section.get('body')
-            if body_format in JSON_SECTIONS and isinstance(section_body, str):
-                try:
-                    section['body'] = json.loads(section_body)
-                except ValueError:
-                    pass
-
-            # Changing tags to a list
-            section['tags'] = tag_dict_to_ai_list(flatten(section.get('tags', {})))
-            if not section['tags']:
-                section.pop('tags')
-
-            # Add the section to the new section array
-            new_sections.append(section)
-
-        # Update result sections
-        result['result']['sections'] = new_sections
-
-        # Rewrite score as verdict
-        result_score = result['result'].pop('score', 0)
-        result['result']['verdict'] = self._get_verdict_from_score(result_score)
-
-        return result
-
-    @elasticapm.capture_span(span_type='datastore')
-    def get_ai_formatted_submission_data(self, sid, user_classification=None,
-                                         cl_engine=forge.get_classification(),
-                                         index_type=None):
-        # Get the submission data
-        submission = self.submission.get(sid, index_type=index_type)
-        if not submission or (user_classification and not cl_engine.is_accessible(
-                user_c12n=user_classification, c12n=submission.classification.value)):
-            return None
-
-        # Auto adjust min_score
-        if submission.max_score < 0:
-            min_score = -1000000
-        else:
-            min_score = 300
-
-        # Parse results
-        results = [
-            self._fix_section_data(r.as_primitives(strip_non_ai_fields=True, strip_null=True), min_score)
-            for r in self.result.multiget(
-                submission.results, as_dictionary=False, error_on_missing=False, index_type=index_type)
-            if min_score <= r.result.score and (not user_classification or cl_engine.is_accessible(
-                user_c12n=user_classification, c12n=r.classification.value))]
-
-        # Create output
-        output = submission.as_primitives(strip_non_ai_fields=True, strip_null=True)
-        output['verdict'] = self._get_verdict_from_score(output.pop('max_score', 0))
-        output['results'] = results
-        return output
-
-    @elasticapm.capture_span(span_type='datastore')
-    def get_ai_formatted_file_results_data(self, sha256, user_classification=None, user_access_control=None,
-                                           cl_engine=forge.get_classification(),
-                                           index_type=None):
-        # Get the submission data
-        file_obj = self.file.get(sha256, index_type=index_type)
-        if not file_obj or (user_classification and not cl_engine.is_accessible(
-                user_c12n=user_classification, c12n=file_obj.classification.value)):
-            return None
-
-        # Check for the max service score
-        items = self.result.search(f"id:{sha256}*", fl="result.score", access_control=user_access_control,
-                                   as_obj=False, rows=1, sort="result.score desc", index_type=index_type)['items']
-        if items:
-            max_score = items[0]['result']['score']
-        else:
-            max_score = 0
-
-        # Auto adjust min_score
-        if max_score < 0:
-            min_score = -1000000
-        else:
-            min_score = 300
-
-        # Get the list of active result keys
-        active_keys, _ = self.list_file_active_keys(
-            sha256, user_access_control, min_score=min_score, index_type=index_type)
-
-        # Parse results
-        results = [
-            self._fix_section_data(r.as_primitives(strip_non_ai_fields=True, strip_null=True), min_score)
-            for r in self.result.multiget(active_keys, as_dictionary=False,
-                                          error_on_missing=False, index_type=index_type)
-            if min_score <= r.result.score and (not user_classification or cl_engine.is_accessible(
-                user_c12n=user_classification, c12n=r.classification.value))]
-
-        # Create output
-        output = file_obj.as_primitives(strip_non_ai_fields=True, strip_null=True)
-        output['verdict'] = self._get_verdict_from_score(max_score)
-        output['results'] = results
-        return output
-
-    @elasticapm.capture_span(span_type='datastore')
-    def list_file_active_keys(self, sha256, access_control=None, min_score=None, index_type=None):
-        query = f"id:{sha256}*"
-        if min_score:
-            query += f" AND result.score:>={min_score}"
-
-        item_list = list(self.result.stream_search(query, fl="id,created,response.service_name,result.score",
-                                                          access_control=access_control, as_obj=False,
-                                                          index_type=index_type))
-
-        item_list.sort(key=lambda k: k["created"], reverse=True)
-
-        active_found = set()
-        active_keys = []
-        alternates = []
-        for item in item_list:
-            if item['response']['service_name'] not in active_found:
-                active_keys.append(item['id'])
-                active_found.add(item['response']['service_name'])
-            else:
-                alternates.append(item)
-
-        return active_keys, alternates
-
-    @elasticapm.capture_span(span_type='datastore')
-    def list_file_childrens(self, sha256, access_control=None):
-        query = f'id:{sha256}* AND response.extracted.sha256:*'
-        service_resp = self.result.grouped_search("response.service_name", query=query, fl='*',
-                                                  sort="created desc", access_control=access_control,
-                                                  as_obj=False)
-
-        output = []
-        processed_sha256 = []
-        for r in service_resp['items']:
-            for extracted in r['items'][0]['response']['extracted']:
-                if extracted['sha256'] not in processed_sha256:
-                    processed_sha256.append(extracted['sha256'])
-                    output.append({
-                        'name': extracted['name'],
-                        'sha256': extracted['sha256']
-                    })
-        return output
-
-    @elasticapm.capture_span(span_type='datastore')
-    def list_file_parents(self, sha256, access_control=None):
-        query = f"response.extracted.sha256:{sha256}"
-        processed_sha256 = []
-        output = []
-
-        response = self.result.search(query, fl='id', sort="created desc",
-                                      access_control=access_control, as_obj=False)
-        for p in response['items']:
-            key = p['id']
-            sha256 = key[:64]
-            if sha256 not in processed_sha256:
-                output.append(key)
-                processed_sha256.append(sha256)
-
-            if len(processed_sha256) >= 10:
-                break
-
-        return output
-
 
 class MetadataValidator:
     """

{assemblyline-4.7.5.dev77 → assemblyline-4.7.5.dev79/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev77
+Version: 4.7.5.dev79
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.5.dev77/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.5.dev77