assemblyline-ui 4.7.0.dev22__py3-none-any.whl → 4.7.0.dev26__py3-none-any.whl

assemblyline_ui/VERSION

@@ -1 +1 @@
-4.7.0.dev22
+4.7.0.dev26

assemblyline_ui/api/v4/file.py

@@ -4,8 +4,6 @@ import re
 import subprocess
 import tempfile
 
-from flask import request
-
 from assemblyline.common.codec import encode_file
 from assemblyline.common.dict_utils import unflatten
 from assemblyline.common.hexdump import dump, hexdump
@@ -15,6 +13,8 @@ from assemblyline.datastore.collection import Index
 from assemblyline.filestore import FileStoreException
 from assemblyline.odm.models.user import ROLES
 from assemblyline.odm.models.user_settings import ENCODINGS as FILE_DOWNLOAD_ENCODINGS
+from flask import request
+
 from assemblyline_ui.api.base import (
     api_login,
     make_api_response,
@@ -48,6 +48,27 @@ file_api._doc = "Perform operations on files"
 API_MAX_SIZE = 1 * 1024 * 1024
 
 
+def retrieve_file_content(file_obj: dict) -> bytes:
+    """ Retrieve file content from filestore or archivestore based on availability and file metadata."""
+    data = None
+    try:
+        if file_obj.get('from_archive'):
+            # Try to download from archive first
+            if ARCHIVESTORE is not None and ARCHIVESTORE.exists(file_obj['sha256']):
+                data = ARCHIVESTORE.get(file_obj['sha256'])
+
+            # If for some reason the file is not in archive, try filestore
+            elif FILESTORE.exists(file_obj['sha256']):
+                data = FILESTORE.get(file_obj['sha256'])
+
+        elif FILESTORE.exists(file_obj['sha256']):
+            # Try to download from filestore
+            data = FILESTORE.get(file_obj['sha256'])
+    except FileStoreException:
+        pass
+
+    return data
+
 @file_api.route("/ascii/<sha256>/", methods=["GET"])
 @api_login(require_role=[ROLES.file_detail])
 def get_file_ascii(sha256, **kwargs):
@@ -71,30 +92,23 @@ def get_file_ascii(sha256, **kwargs):
     """
 
     user = kwargs['user']
-    file_obj = STORAGE.file.get(sha256, as_obj=False)
+    index_type = None
+    if ROLES.archive_download in user['roles']:
+        # User is allowed to access archive, so we check both hot and archive
+        index_type = Index.HOT_AND_ARCHIVE
+
+    file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
 
-    if not file_obj or not FILESTORE.exists(sha256):
+    if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
-        try:
-            data = FILESTORE.get(sha256)[:API_MAX_SIZE]
-        except (FileStoreException, TypeError):
-            data = None
-
-        # Try to download from archive
-        if not data and \
-                ARCHIVESTORE is not None and \
-                ARCHIVESTORE != FILESTORE and \
-                ROLES.archive_download in user['roles'] and \
-                ARCHIVESTORE.exists(sha256):
-            try:
-                data = ARCHIVESTORE.get(sha256)
-            except FileStoreException:
-                data = None
-
+        data = retrieve_file_content(file_obj)
         if not data:
             return make_api_response({}, "This file was not found in the system.", 404)
+        else:
+            # Trim data to API max size
+            data = data[:API_MAX_SIZE]
 
         return make_api_response({
             "content": data.translate(FILTER_ASCII).decode(),
@@ -129,7 +143,12 @@ def download_file(sha256, **kwargs):
     <THE FILE BINARY ENCODED IN SPECIFIED FORMAT>
     """
     user = kwargs['user']
-    file_obj = STORAGE.file.get(sha256, as_obj=False)
+    index_type = None
+    if ROLES.archive_download in user['roles']:
+        # User is allowed to access archive, so we check both hot and archive
+        index_type = Index.HOT_AND_ARCHIVE
+
+    file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
 
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
@@ -145,7 +164,7 @@ def download_file(sha256, **kwargs):
         submission = {}
         file_metadata = {}
         if sid is not None:
-            submission = STORAGE.submission.get(sid, as_obj=False)
+            submission = STORAGE.submission.get(sid, as_obj=False, index_type=index_type)
             if submission is None:
                 submission = {}
 
@@ -185,20 +204,21 @@ def download_file(sha256, **kwargs):
             _, download_path = tempfile.mkstemp()
 
         try:
+            downloaded_from = None
             try:
-                downloaded_from = FILESTORE.download(sha256, download_path)
+                if file_obj.get('from_archive'):
+                    # Try to download from archive first
+                    if ARCHIVESTORE is not None and ARCHIVESTORE.exists(sha256):
+                        downloaded_from = ARCHIVESTORE.download(sha256, download_path)
+
+                    # If for some reason the file is not in archive, try filestore
+                    elif FILESTORE.exists(sha256):
+                        downloaded_from = FILESTORE.download(sha256, download_path)
+                else:
+                    # Try to download from filestore
+                    downloaded_from = FILESTORE.download(sha256, download_path)
             except FileStoreException:
-                downloaded_from = None
-
-            # Try to download from archive
-            if not downloaded_from and \
-                    ARCHIVESTORE is not None and \
-                    ARCHIVESTORE != FILESTORE and \
-                    ROLES.archive_download in user['roles']:
-                try:
-                    downloaded_from = ARCHIVESTORE.download(sha256, download_path)
-                except FileStoreException:
-                    downloaded_from = None
+                pass
 
             if not downloaded_from:
                 return make_api_response({}, "The file was not found in the system.", 404)
@@ -379,7 +399,12 @@ def summarize_code_snippet(sha256, **kwargs):
         ai_summary = CACHE.get(cache_key)
 
     if not ai_summary:
-        file_obj = STORAGE.file.get(sha256, as_obj=False)
+        index_type = None
+        if ROLES.archive_download in user['roles']:
+            # User is allowed to access archive, so we check both hot and archive
+            index_type = Index.HOT_AND_ARCHIVE
+
+        file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
 
         if not file_obj:
             return make_api_response({}, "The file was not found in the system.", 404)
@@ -392,22 +417,7 @@ def summarize_code_snippet(sha256, **kwargs):
         #     return make_api_response({}, "This file is too big to be seen through this API.", 403)
 
         if user and Classification.is_accessible(user['classification'], file_obj['classification']):
-            try:
-                data = FILESTORE.get(sha256)
-            except (FileStoreException, TypeError):
-                data = None
-
-            # Try to download from archive
-            if not data and \
-                    ARCHIVESTORE is not None and \
-                    ARCHIVESTORE != FILESTORE and \
-                    ROLES.archive_download in user['roles'] and \
-                    ARCHIVESTORE.exists(sha256):
-                try:
-                    data = ARCHIVESTORE.get(sha256)
-                except FileStoreException:
-                    data = None
-
+            data = retrieve_file_content(file_obj)
             if not data:
                 return make_api_response({}, "The file was not found in the system.", 404)
 
@@ -450,33 +460,30 @@ def get_file_hex(sha256, **kwargs):
     }
     """
     user = kwargs['user']
-    file_obj = STORAGE.file.get(sha256, as_obj=False)
+    index_type = None
+    if ROLES.archive_download in user['roles']:
+        # User is allowed to access archive, so we check both hot and archive
+        index_type = Index.HOT_AND_ARCHIVE
+    file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
 
     bytes_only = request.args.get('bytes_only', 'false').lower() in ['true', '']
     length = int(request.args.get('length', '16'))
 
-    if not file_obj or not FILESTORE.exists(sha256):
+    if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
+        data = retrieve_file_content(file_obj)
         try:
             data = FILESTORE.get(sha256)[:API_MAX_SIZE]
         except FileStoreException:
             data = None
 
-        # Try to download from archive
-        if not data and \
-                ARCHIVESTORE is not None and \
-                ARCHIVESTORE != FILESTORE and \
-                ROLES.archive_download in user['roles'] and \
-                ARCHIVESTORE.exists(sha256):
-            try:
-                data = ARCHIVESTORE.get(sha256)
-            except FileStoreException:
-                data = None
-
         if not data:
             return make_api_response({}, "This file was not found in the system.", 404)
+        else:
+            # Trim data to API max size
+            data = data[:API_MAX_SIZE]
 
         if bytes_only:
             return make_api_response({
@@ -514,7 +521,11 @@ def get_file_image_datastream(sha256, **kwargs):
     data:image/png;base64,...
     """
     user = kwargs['user']
-    file_obj = STORAGE.file.get(sha256, as_obj=False)
+    index_type = None
+    if ROLES.archive_download in user['roles']:
+        # User is allowed to access archive, so we check both hot and archive
+        index_type = Index.HOT_AND_ARCHIVE
+    file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
 
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
@@ -523,22 +534,7 @@ def get_file_image_datastream(sha256, **kwargs):
         return make_api_response({}, "This file is not allowed to be downloaded as a datastream.", 403)
 
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
-        try:
-            data = FILESTORE.get(sha256)
-        except (FileStoreException, TypeError):
-            data = None
-
-        # Try to download from archive
-        if not data and \
-                ARCHIVESTORE is not None and \
-                ARCHIVESTORE != FILESTORE and \
-                ROLES.archive_download in user['roles'] and \
-                ARCHIVESTORE.exists(sha256):
-            try:
-                data = ARCHIVESTORE.get(sha256)
-            except FileStoreException:
-                data = None
-
+        data = retrieve_file_content(file_obj)
         if not data:
             return make_api_response({}, "This file was not found in the system.", 404)
 
@@ -570,30 +566,23 @@ def get_file_strings(sha256, **kwargs):
     """
     user = kwargs['user']
     hlen = request.args.get('len', "6")
-    file_obj = STORAGE.file.get(sha256, as_obj=False)
+    index_type = None
+    if ROLES.archive_download in user['roles']:
+        # User is allowed to access archive, so we check both hot and archive
+        index_type = Index.HOT_AND_ARCHIVE
 
-    if not file_obj or not FILESTORE.exists(sha256):
+    file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
+
+    if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
-        try:
-            data = FILESTORE.get(sha256)[:API_MAX_SIZE]
-        except (FileStoreException, TypeError):
-            data = None
-
-        # Try to download from archive
-        if not data and \
-                ARCHIVESTORE is not None and \
-                ARCHIVESTORE != FILESTORE and \
-                ROLES.archive_download in user['roles'] and \
-                ARCHIVESTORE.exists(sha256):
-            try:
-                data = ARCHIVESTORE.get(sha256)
-            except FileStoreException:
-                data = None
-
+        data = retrieve_file_content(file_obj)
         if not data:
             return make_api_response({}, "This file was not found in the system.", 404)
+        else:
+            # Trim data to API max size
+            data = data[:API_MAX_SIZE]
 
         # Ascii strings (we use decode with replace on to create delimiters)
         pattern = "[\x20-\x7e]{%s,}" % hlen
@@ -741,11 +730,9 @@ def get_file_results(sha256, **kwargs):
      "file_viewer_only": True }  # UI switch to disable features
     """
     user = kwargs['user']
-
+    index_type = None
     if str(request.args.get('archive_only', 'false')).lower() in ['true', '']:
         index_type = Index.ARCHIVE
-    else:
-        index_type = None
 
     file_obj = STORAGE.file.get(sha256, as_obj=False, index_type=index_type)
 

assemblyline_ui/api/v4/ui.py

@@ -3,16 +3,16 @@
 
 import os
 
-from assemblyline_core.submission_client import SubmissionClient, SubmissionException
-from cart import get_metadata_only, is_cart
-from flask import request
-
 from assemblyline.common import forge
 from assemblyline.common.bundling import import_bundle
 from assemblyline.common.str_utils import safe_str
 from assemblyline.common.uid import get_random_id
 from assemblyline.odm.messages.submission import Submission
 from assemblyline.odm.models.user import ROLES
+from assemblyline_core.submission_client import SubmissionClient, SubmissionException
+from cart import get_metadata_only, is_cart
+from flask import request
+
 from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint
 from assemblyline_ui.config import (
     FILESTORE,
@@ -249,7 +249,8 @@ def start_ui_submission(ui_sid, **kwargs):
                     meta = get_metadata_only(submitted_file)
                     if meta.get('al', {}).get('type', 'unknown') == 'archive/bundle/al':
                         try:
-                            submission = import_bundle(submitted_file, allow_incomplete=True, identify=IDENTIFY)
+                            submission = import_bundle(submitted_file, allow_incomplete=True, identify=IDENTIFY,
+                                                       dtl=ui_params.get('ttl'))
                         except Exception as e:
                             return make_api_response("", err=str(e), status_code=400)
                         return make_api_response({"started": True, "sid": submission['sid']})

assemblyline_ui/api/v4/user.py

@@ -198,7 +198,7 @@ def who_am_i(**kwargs):
     # System configuration
     user_data['c12nDef'] = classification_definition
     user_data['classification_aliases'] = {k: v for k, v in CLASSIFICATION_ALIASES.items().items()
-                                           if k in user_data['classification'] or user_data['is_admin']}
+                                           if k in user_data['classification'] or user_data['is_admin'] or CLASSIFICATION.is_accessible(user_data['classification'], CLASSIFICATION.RESTRICTED)}
 
     # create tag-to-source lookup mapping
     external_source_tags = {}

{assemblyline_ui-4.7.0.dev22.dist-info → assemblyline_ui-4.7.0.dev26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-ui
-Version: 4.7.0.dev22
+Version: 4.7.0.dev26
 Summary: Assemblyline 4 - API and Socket IO server
 Home-page: https://github.com/CybercentreCanada/assemblyline-ui/
 Author: CCCS Assemblyline development team

{assemblyline_ui-4.7.0.dev22.dist-info → assemblyline_ui-4.7.0.dev26.dist-info}/RECORD

@@ -1,4 +1,4 @@
-assemblyline_ui/VERSION,sha256=wTj4QG6gYDXtgPv-jUiiUcMYe5HAQ0e1rxS_R8jpIA0,12
+assemblyline_ui/VERSION,sha256=hlFS3gZZIB9hMecU1bN5qHS-QUPk60zLIau9FATXyoI,12
 assemblyline_ui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_ui/app.py,sha256=ok96rpL0-Hb5amqa7ojhuB0boQceZG7E32NX6LZm-f4,7849
 assemblyline_ui/config.py,sha256=4M5wpG_OwJPjZAjRWIUMMyF4KG30oJidNDnFM-IVJCw,8927
@@ -22,7 +22,7 @@ assemblyline_ui/api/v4/badlist.py,sha256=lTqRLs8F745U0biPdC4AHimj0VUiCLcNFEae7Un
 assemblyline_ui/api/v4/bundle.py,sha256=UcLUpErpiAk5LSYeC0zHDw_WpQiPcM8uhH7NwmWi4hw,5521
 assemblyline_ui/api/v4/error.py,sha256=RdpnDqM-tiFdG-Qni3-_8kn7VPlB0UW1VafUF2EHw8Q,2586
 assemblyline_ui/api/v4/federated_lookup.py,sha256=aq7KMBu1x6ZTvSOfSq_L0Y-Q6VFIxsE8IVK0BZ9w0y8,12540
-assemblyline_ui/api/v4/file.py,sha256=rHBgN0XaLmuUf64-Z8ca9vHrS_av3VbQRtALXGsDlxY,40009
+assemblyline_ui/api/v4/file.py,sha256=Uut6miKNW5DvSMwDoSVmujerG9-KcywPAe68OqtfPRs,40116
 assemblyline_ui/api/v4/hash_search.py,sha256=2KidEgffDWoMJzmFC49dN6OctcsjdRo-lNVgx1GorOA,12129
 assemblyline_ui/api/v4/help.py,sha256=HmCJfBZXv5DM1e_ZEtjq_83UIr-jUyCNtLueXboUAWg,6767
 assemblyline_ui/api/v4/heuristics.py,sha256=YfTUmDYrilwdNO1Dlr0qfDRTU4fOhPpSCWARLjt1auo,2910
@@ -40,8 +40,8 @@ assemblyline_ui/api/v4/signature.py,sha256=av3FhYkIuXjKX91DX93WcN_U5L2bltDHJSwRb
 assemblyline_ui/api/v4/submission.py,sha256=WxMY00ePNdOMFXyu9j38sxpe0vT26u-Z_biMiBUOxds,46345
 assemblyline_ui/api/v4/submit.py,sha256=5nfuX6Ud3FMdDGIOgBAdcb6kpMbINk750Ph5yp90O8c,18372
 assemblyline_ui/api/v4/system.py,sha256=4JhLhpmPfKzbXiHsBoJxSZP-ibu-lnmk4FrtaLExaRM,24861
-assemblyline_ui/api/v4/ui.py,sha256=IlMR77TIgIdoWitKK1CLU20X27x1IxnMFbWgyLVGH4M,11427
-assemblyline_ui/api/v4/user.py,sha256=AlSVtbKgPlNtlyKnmz4fyDYgcpD5uBSq-dRI83L3yEY,47760
+assemblyline_ui/api/v4/ui.py,sha256=J3OoLEETlFgeq7DbSlubTPebMQJOUx6UwRIt8mCBYbQ,11508
+assemblyline_ui/api/v4/user.py,sha256=m_xvc59imrxbRBwnNkZeH2vZbcW5smDgLIcIRVobaP0,47848
 assemblyline_ui/api/v4/webauthn.py,sha256=ouVWMMd0aMnNV5iaKbWY1AnsUE5_BKAtaUg9j77Rvow,4875
 assemblyline_ui/api/v4/workflow.py,sha256=Uz50960KyvFDm39NKYNSq5OXT0CA0u_FO5xPP61aYiQ,10579
 assemblyline_ui/helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -74,8 +74,8 @@ assemblyline_ui/sio/live_submission.py,sha256=mS0oGO5rEA8PLa8kjBcodOG-Q0CG4SpYAo
 assemblyline_ui/sio/retrohunt.py,sha256=vlH_x6Rb4pm3GM1sNLH1C42-NVR3VoAsEfkAu3L30fY,2146
 assemblyline_ui/sio/status.py,sha256=_Bxf1KLPOJEUIk4J9_j9fzvQWUXqSIT9xnJKhT1hhuc,1980
 assemblyline_ui/sio/submission.py,sha256=IYJuGz73HK9HtYfqc-gWW8tc1lt5VZ62Qn64AaGGtm8,2222
-assemblyline_ui-4.7.0.dev22.dist-info/licenses/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_ui-4.7.0.dev22.dist-info/METADATA,sha256=lFO81CE1lyzmVUEGKLAr_XPKRBx870gphNsaJmSU7Gs,8174
-assemblyline_ui-4.7.0.dev22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-assemblyline_ui-4.7.0.dev22.dist-info/top_level.txt,sha256=WLa7-PKLJTbMUbKKU3q3kg5_uAV67hss5kC71PAbIeg,16
-assemblyline_ui-4.7.0.dev22.dist-info/RECORD,,
+assemblyline_ui-4.7.0.dev26.dist-info/licenses/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_ui-4.7.0.dev26.dist-info/METADATA,sha256=BitVBEa6wa8yC1rYZMLf1QGRQvlEAgXeVg24mCyipz8,8174
+assemblyline_ui-4.7.0.dev26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+assemblyline_ui-4.7.0.dev26.dist-info/top_level.txt,sha256=WLa7-PKLJTbMUbKKU3q3kg5_uAV67hss5kC71PAbIeg,16
+assemblyline_ui-4.7.0.dev26.dist-info/RECORD,,