assemblyline 4.8.1.dev1__tar.gz → 4.8.1.dev2__tar.gz

{assemblyline-4.8.1.dev1/assemblyline.egg-info → assemblyline-4.8.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.8.1.dev1
+Version: 4.8.1.dev2
 Summary: Assemblyline 4 - Automated malware analysis framework
 Author-email: CCCS Assemblyline development team <assemblyline@cyber.gc.ca>
 License-Expression: MIT
@@ -17,9 +17,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENCE.md
 Requires-Dist: arrow
-Requires-Dist: aiohttp
 Requires-Dist: lark
-Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
 Requires-Dist: sftpretty
@@ -35,7 +33,6 @@ Requires-Dist: assemblyline-toolbox>=0.4.0
 Requires-Dist: python-magic
 Requires-Dist: pytz
 Requires-Dist: apscheduler
-Requires-Dist: websocket_client<1.0.0
 Requires-Dist: elastic-apm[flask]>=6.13.0
 Requires-Dist: notifications-python-client
 Requires-Dist: rstr
@@ -51,7 +48,8 @@ Requires-Dist: pytest; extra == "test"
 Requires-Dist: retrying; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: pyftpdlib; extra == "test"
-Requires-Dist: pyopenssl==23.3.0; extra == "test"
+Requires-Dist: pyopenssl; extra == "test"
+Requires-Dist: cryptography; extra == "test"
 Dynamic: license-file
 
 [![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu)

assemblyline-4.8.1.dev2/assemblyline/VERSION

@@ -0,0 +1 @@
+4.8.1.dev2

{assemblyline-4.8.1.dev1 → assemblyline-4.8.1.dev2}/assemblyline/cachestore/__init__.py

@@ -49,7 +49,7 @@ class CacheStore(object):
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
 
         self.datastore.cached_file.save(new_key, {'expiry_ts': now_as_iso(ttl), 'component': self.component})
-        self.filestore.upload(new_key, path, force=True)
+        self.filestore.upload(path, new_key, force=True)
 
     def touch(self, cache_key: str, ttl=DEFAULT_CACHE_LEN):
         if not COMPONENT_VALIDATOR.match(cache_key):

{assemblyline-4.8.1.dev1 → assemblyline-4.8.1.dev2}/assemblyline/common/identify.py

@@ -41,6 +41,10 @@ CUSTOM_PS1_ID = b"#!/usr/bin/env pwsh\n"
 CUSTOM_BATCH_ID = b"REM Batch extracted by Assemblyline\n"
 CUSTOM_URI_ID = "# Assemblyline URI file\n"
 
+PDF_ENCRYPT_MARKER = b"/Encrypt"
+PDF_PORTFOLIO_MARKER = b"/Type/Catalog/Collection"
+PDF_SCAN_BLOCKSIZE = 1048576
+
 
 class Identify:
     def __init__(self, use_cache: bool = True, config=None, datastore=None, log=None) -> None:
@@ -452,13 +456,9 @@ class Identify:
         #  - Check for encryption
         #  - Check for PDF collection (portfolio)
         if data["type"] == "document/pdf":
-            # Password protected documents typically contain '/Encrypt'
-            pdf_content = open(path, "rb").read()
-            if re.search(b"/Encrypt", pdf_content):
-                data["type"] = "document/pdf/passwordprotected"
-            # Portfolios typically contain '/Type/Catalog/Collection
-            elif re.search(b"/Type/Catalog/Collection", pdf_content):
-                data["type"] = "document/pdf/portfolio"
+            refined_type = pdf_ident(path)
+            if refined_type:
+                data["type"] = refined_type
 
         return data
 
@@ -640,6 +640,28 @@ def uri_ident(path: str, info: Dict) -> str:
     return f"uri/{u.scheme}"
 
 
+def pdf_ident(path: str) -> Optional[str]:
+    # The file is scanned in fixed-size blocks (with a small overlap so markers straddling a
+    # block boundary are still found) to keep memory usage flat no matter the file size.
+    overlap = len(PDF_PORTFOLIO_MARKER) - 1
+    saw_portfolio = False
+    tail = b""
+    with open(path, "rb") as fh:
+        while block := fh.read(PDF_SCAN_BLOCKSIZE):
+            boundary = tail + block[:overlap]
+            # Password protected documents typically contain '/Encrypt'
+            # '/Encrypt' anywhere in the file takes precedence over the portfolio marker.
+            if PDF_ENCRYPT_MARKER in block or PDF_ENCRYPT_MARKER in boundary:
+                return "document/pdf/passwordprotected"
+            # Portfolios typically contain '/Type/Catalog/Collection'
+            if not saw_portfolio and (PDF_PORTFOLIO_MARKER in block or PDF_PORTFOLIO_MARKER in boundary):
+                saw_portfolio = True
+            tail = block[-overlap:]
+
+    if saw_portfolio:
+        return "document/pdf/portfolio"
+    return None
+
 def shell_main():
     from pprint import pprint
 

{assemblyline-4.8.1.dev1 → assemblyline-4.8.1.dev2}/assemblyline/common/tag_safelist.yml

@@ -92,11 +92,10 @@ regex:
     - https?://schemas\.android\.com/apk/res(-auto|/android)
     - https?://xmlpull\.org/v1/doc/features\.html(?:$|.*)
     - https?://android\.googlesource\.com/toolchain/llvm-project
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/relationships/(image|attachedTemplate|header|footnotes|fontTable|customXml|endnotes|theme|settings|webSettings|glossaryDocument|numbering|footer|styles)
+    - ^https?://schemas\.openxmlformats\.org/officeDocument/2006/(?:math|relationships(?:/[\w./-]+)?)$
     - https?://schemas\.openxmlformats\.org/drawingml/2006/(main|wordprocessingDrawing)
-    - https?://schemas\.openxmlformats\.org/package/2006/relationships
+    - ^https?://schemas\.openxmlformats\.org/package/2006/relationships(?:/[\w./-]+)?$
     - https?://schemas\.openxmlformats\.org/markup-compatibility/2006
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/(relationships|math)
     - https?://schemas\.openxmlformats\.org/word/2010/wordprocessingShape
     - https?://schemas\.openxmlformats\.org/wordprocessingml/2006/main
     - https?://www\.verisign\.com/(rpa0|rpa|cps0)
@@ -117,9 +116,9 @@ regex:
   # Azure
     - https?://wpad\..*/wpad\.dat
   # Google
-    - https?://www\.google\.c(om|a)
+    - ^https?://www\.google\.c(?:om|a)/?$
   # Adobe
-    - (?:https?://)?files\.acrobat\.com(?:(?::443)|/?$)
+    - ^(?:https?://)?files\.acrobat\.com(?::443)?/?$
   network.static.uri:
     - ^(\w{2,5}:\/\/)?[^:\/]*\.verisign\.com(:([1-9]|[1-5]?[0-9]{2,4}|6[1-4][0-9]{3}|65[1-4][0-9]{2}|655[1-2][0-9]|6553[1-5]))?\/[^:]*$
     - ^(\w{2,5}:\/\/)?[^:\/]*\.bootstrapcdn\.com(:([1-9]|[1-5]?[0-9]{2,4}|6[1-4][0-9]{3}|65[1-4][0-9]{2}|655[1-2][0-9]|6553[1-5]))?\/[^:]*$
@@ -129,16 +128,15 @@ regex:
     - https?://schemas\.android\.com/apk/res(-auto|/android)
     - https?://xmlpull\.org/v1/doc/features\.html(?:$|.*)
     - https?://android\.googlesource\.com/toolchain/llvm-project
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/relationships/(image|attachedTemplate|header|footnotes|fontTable|customXml|endnotes|theme|settings|webSettings|glossaryDocument|numbering|footer|styles)
+    - ^https?://schemas\.openxmlformats\.org/officeDocument/2006/(?:math|relationships(?:/[\w./-]+)?)$
     - https?://schemas\.microsoft\.com/office/word/2010/(wordml|wordprocessingCanvas|wordprocessingInk|wordprocessingGroup|wordprocessingDrawing)
     - https?://schemas\.microsoft\.com/office/word/(2012|2006)/wordml
     - https?://schemas\.microsoft\.com/office/word/2015/wordml/symex
     - https?://schemas\.microsoft\.com/office/drawing/2014/chartex
     - https?://schemas\.microsoft\.com/office/drawing/2015/9/8/chartex
     - https?://schemas\.openxmlformats\.org/drawingml/2006/(main|wordprocessingDrawing)
-    - https?://schemas\.openxmlformats\.org/package/2006/relationships
+    - ^https?://schemas\.openxmlformats\.org/package/2006/relationships(?:/[\w./-]+)?$
     - https?://schemas\.openxmlformats\.org/markup-compatibility/2006
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/(relationships|math)
     - https?://schemas\.openxmlformats\.org/word/2010/wordprocessingShape
     - https?://schemas\.openxmlformats\.org/wordprocessingml/2006/main
     - https?://www\.verisign\.com/(rpa0|rpa|cps0)
@@ -247,9 +245,9 @@ regex:
     - ((api|www|platform)\.)?bing\.com$
   # Azure
     - md-ssd-.+\.blob\.core\.windows\.net$
-    - .+\.table\.core\.windows\.net
-    - .+\.blob\.core\.windows\.net
-    - .+\.opinsights\.azure\.com
+    - .+\.table\.core\.windows\.net$
+    - .+\.blob\.core\.windows\.net$
+    - .+\.opinsights\.azure\.com$
     - agentserviceapi\.azure-automation\.net$
     - agentserviceapi\.guestconfiguration\.azure\.com$
     - .+\.blob\.storage\.azure\.net$

{assemblyline-4.8.1.dev1 → assemblyline-4.8.1.dev2/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.8.1.dev1
+Version: 4.8.1.dev2
 Summary: Assemblyline 4 - Automated malware analysis framework
 Author-email: CCCS Assemblyline development team <assemblyline@cyber.gc.ca>
 License-Expression: MIT
@@ -17,9 +17,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENCE.md
 Requires-Dist: arrow
-Requires-Dist: aiohttp
 Requires-Dist: lark
-Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
 Requires-Dist: sftpretty
@@ -35,7 +33,6 @@ Requires-Dist: assemblyline-toolbox>=0.4.0
 Requires-Dist: python-magic
 Requires-Dist: pytz
 Requires-Dist: apscheduler
-Requires-Dist: websocket_client<1.0.0
 Requires-Dist: elastic-apm[flask]>=6.13.0
 Requires-Dist: notifications-python-client
 Requires-Dist: rstr
@@ -51,7 +48,8 @@ Requires-Dist: pytest; extra == "test"
 Requires-Dist: retrying; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: pyftpdlib; extra == "test"
-Requires-Dist: pyopenssl==23.3.0; extra == "test"
+Requires-Dist: pyopenssl; extra == "test"
+Requires-Dist: cryptography; extra == "test"
 Dynamic: license-file
 
 [![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu)

{assemblyline-4.8.1.dev1 → assemblyline-4.8.1.dev2}/assemblyline.egg-info/requires.txt

@@ -1,7 +1,5 @@
 arrow
-aiohttp
 lark
-urllib3>=2.6.0
 python-baseconv
 boto3
 sftpretty
@@ -17,7 +15,6 @@ assemblyline-toolbox>=0.4.0
 python-magic
 pytz
 apscheduler
-websocket_client<1.0.0
 elastic-apm[flask]>=6.13.0
 notifications-python-client
 rstr
@@ -34,4 +31,5 @@ pytest
 retrying
 pytest-mock
 pyftpdlib
-pyopenssl==23.3.0
+pyopenssl
+cryptography

{assemblyline-4.8.1.dev1 → assemblyline-4.8.1.dev2}/pyproject.toml

@@ -36,9 +36,7 @@ classifiers = [
 ]
 dependencies = [
     "arrow",
-    "aiohttp",
     "lark",
-    "urllib3>=2.6.0",
     "python-baseconv",
     "boto3",
     "sftpretty",
@@ -54,7 +52,6 @@ dependencies = [
     "python-magic",
     "pytz",
     "apscheduler",
-    "websocket_client<1.0.0",
     "elastic-apm[flask]>=6.13.0",
     "notifications-python-client",
     "rstr",
@@ -77,7 +74,8 @@ test = [
     "retrying",
     "pytest-mock",
     "pyftpdlib",
-    "pyopenssl==23.3.0",
+    "pyopenssl",
+    "cryptography",
 ]
 
 [project.scripts]

assemblyline-4.8.1.dev1/assemblyline/VERSION

@@ -1 +0,0 @@
-4.8.1.dev1