assemblyline 4.8.1.dev0__tar.gz → 4.8.1.dev2__tar.gz

{assemblyline-4.8.1.dev0/assemblyline.egg-info → assemblyline-4.8.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.8.1.dev0
+Version: 4.8.1.dev2
 Summary: Assemblyline 4 - Automated malware analysis framework
 Author-email: CCCS Assemblyline development team <assemblyline@cyber.gc.ca>
 License-Expression: MIT
@@ -17,12 +17,10 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENCE.md
 Requires-Dist: arrow
-Requires-Dist: aiohttp
 Requires-Dist: lark
-Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
-Requires-Dist: pysftp
+Requires-Dist: sftpretty
 Requires-Dist: redis
 Requires-Dist: requests[socks]
 Requires-Dist: elasticsearch<9.0.0,>=8.0.0
@@ -35,7 +33,6 @@ Requires-Dist: assemblyline-toolbox>=0.4.0
 Requires-Dist: python-magic
 Requires-Dist: pytz
 Requires-Dist: apscheduler
-Requires-Dist: websocket_client<1.0.0
 Requires-Dist: elastic-apm[flask]>=6.13.0
 Requires-Dist: notifications-python-client
 Requires-Dist: rstr
@@ -46,13 +43,13 @@ Requires-Dist: chardet<6
 Requires-Dist: yara-python
 Requires-Dist: hauntedhouse==0.1.10
 Requires-Dist: magika
-Requires-Dist: paramiko<4
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: retrying; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: pyftpdlib; extra == "test"
-Requires-Dist: pyopenssl==23.3.0; extra == "test"
+Requires-Dist: pyopenssl; extra == "test"
+Requires-Dist: cryptography; extra == "test"
 Dynamic: license-file
 
 [![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu)

assemblyline-4.8.1.dev2/assemblyline/VERSION

@@ -0,0 +1 @@
+4.8.1.dev2

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2}/assemblyline/cachestore/__init__.py

@@ -49,7 +49,7 @@ class CacheStore(object):
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
 
         self.datastore.cached_file.save(new_key, {'expiry_ts': now_as_iso(ttl), 'component': self.component})
-        self.filestore.upload(new_key, path, force=True)
+        self.filestore.upload(path, new_key, force=True)
 
     def touch(self, cache_key: str, ttl=DEFAULT_CACHE_LEN):
         if not COMPONENT_VALIDATOR.match(cache_key):

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2}/assemblyline/common/identify.py

@@ -41,6 +41,10 @@ CUSTOM_PS1_ID = b"#!/usr/bin/env pwsh\n"
 CUSTOM_BATCH_ID = b"REM Batch extracted by Assemblyline\n"
 CUSTOM_URI_ID = "# Assemblyline URI file\n"
 
+PDF_ENCRYPT_MARKER = b"/Encrypt"
+PDF_PORTFOLIO_MARKER = b"/Type/Catalog/Collection"
+PDF_SCAN_BLOCKSIZE = 1048576
+
 
 class Identify:
     def __init__(self, use_cache: bool = True, config=None, datastore=None, log=None) -> None:
@@ -452,13 +456,9 @@ class Identify:
         #  - Check for encryption
         #  - Check for PDF collection (portfolio)
         if data["type"] == "document/pdf":
-            # Password protected documents typically contain '/Encrypt'
-            pdf_content = open(path, "rb").read()
-            if re.search(b"/Encrypt", pdf_content):
-                data["type"] = "document/pdf/passwordprotected"
-            # Portfolios typically contain '/Type/Catalog/Collection
-            elif re.search(b"/Type/Catalog/Collection", pdf_content):
-                data["type"] = "document/pdf/portfolio"
+            refined_type = pdf_ident(path)
+            if refined_type:
+                data["type"] = refined_type
 
         return data
 
@@ -640,6 +640,28 @@ def uri_ident(path: str, info: Dict) -> str:
     return f"uri/{u.scheme}"
 
 
+def pdf_ident(path: str) -> Optional[str]:
+    # The file is scanned in fixed-size blocks (with a small overlap so markers straddling a
+    # block boundary are still found) to keep memory usage flat no matter the file size.
+    overlap = len(PDF_PORTFOLIO_MARKER) - 1
+    saw_portfolio = False
+    tail = b""
+    with open(path, "rb") as fh:
+        while block := fh.read(PDF_SCAN_BLOCKSIZE):
+            boundary = tail + block[:overlap]
+            # Password protected documents typically contain '/Encrypt'
+            # '/Encrypt' anywhere in the file takes precedence over the portfolio marker.
+            if PDF_ENCRYPT_MARKER in block or PDF_ENCRYPT_MARKER in boundary:
+                return "document/pdf/passwordprotected"
+            # Portfolios typically contain '/Type/Catalog/Collection'
+            if not saw_portfolio and (PDF_PORTFOLIO_MARKER in block or PDF_PORTFOLIO_MARKER in boundary):
+                saw_portfolio = True
+            tail = block[-overlap:]
+
+    if saw_portfolio:
+        return "document/pdf/portfolio"
+    return None
+
 def shell_main():
     from pprint import pprint
 

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2}/assemblyline/common/tag_safelist.yml

@@ -92,11 +92,10 @@ regex:
     - https?://schemas\.android\.com/apk/res(-auto|/android)
     - https?://xmlpull\.org/v1/doc/features\.html(?:$|.*)
     - https?://android\.googlesource\.com/toolchain/llvm-project
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/relationships/(image|attachedTemplate|header|footnotes|fontTable|customXml|endnotes|theme|settings|webSettings|glossaryDocument|numbering|footer|styles)
+    - ^https?://schemas\.openxmlformats\.org/officeDocument/2006/(?:math|relationships(?:/[\w./-]+)?)$
     - https?://schemas\.openxmlformats\.org/drawingml/2006/(main|wordprocessingDrawing)
-    - https?://schemas\.openxmlformats\.org/package/2006/relationships
+    - ^https?://schemas\.openxmlformats\.org/package/2006/relationships(?:/[\w./-]+)?$
     - https?://schemas\.openxmlformats\.org/markup-compatibility/2006
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/(relationships|math)
     - https?://schemas\.openxmlformats\.org/word/2010/wordprocessingShape
     - https?://schemas\.openxmlformats\.org/wordprocessingml/2006/main
     - https?://www\.verisign\.com/(rpa0|rpa|cps0)
@@ -117,9 +116,9 @@ regex:
   # Azure
     - https?://wpad\..*/wpad\.dat
   # Google
-    - https?://www\.google\.c(om|a)
+    - ^https?://www\.google\.c(?:om|a)/?$
   # Adobe
-    - (?:https?://)?files\.acrobat\.com(?:(?::443)|/?$)
+    - ^(?:https?://)?files\.acrobat\.com(?::443)?/?$
   network.static.uri:
     - ^(\w{2,5}:\/\/)?[^:\/]*\.verisign\.com(:([1-9]|[1-5]?[0-9]{2,4}|6[1-4][0-9]{3}|65[1-4][0-9]{2}|655[1-2][0-9]|6553[1-5]))?\/[^:]*$
     - ^(\w{2,5}:\/\/)?[^:\/]*\.bootstrapcdn\.com(:([1-9]|[1-5]?[0-9]{2,4}|6[1-4][0-9]{3}|65[1-4][0-9]{2}|655[1-2][0-9]|6553[1-5]))?\/[^:]*$
@@ -129,16 +128,15 @@ regex:
     - https?://schemas\.android\.com/apk/res(-auto|/android)
     - https?://xmlpull\.org/v1/doc/features\.html(?:$|.*)
     - https?://android\.googlesource\.com/toolchain/llvm-project
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/relationships/(image|attachedTemplate|header|footnotes|fontTable|customXml|endnotes|theme|settings|webSettings|glossaryDocument|numbering|footer|styles)
+    - ^https?://schemas\.openxmlformats\.org/officeDocument/2006/(?:math|relationships(?:/[\w./-]+)?)$
     - https?://schemas\.microsoft\.com/office/word/2010/(wordml|wordprocessingCanvas|wordprocessingInk|wordprocessingGroup|wordprocessingDrawing)
     - https?://schemas\.microsoft\.com/office/word/(2012|2006)/wordml
     - https?://schemas\.microsoft\.com/office/word/2015/wordml/symex
     - https?://schemas\.microsoft\.com/office/drawing/2014/chartex
     - https?://schemas\.microsoft\.com/office/drawing/2015/9/8/chartex
     - https?://schemas\.openxmlformats\.org/drawingml/2006/(main|wordprocessingDrawing)
-    - https?://schemas\.openxmlformats\.org/package/2006/relationships
+    - ^https?://schemas\.openxmlformats\.org/package/2006/relationships(?:/[\w./-]+)?$
     - https?://schemas\.openxmlformats\.org/markup-compatibility/2006
-    - https?://schemas\.openxmlformats\.org/officeDocument/2006/(relationships|math)
     - https?://schemas\.openxmlformats\.org/word/2010/wordprocessingShape
     - https?://schemas\.openxmlformats\.org/wordprocessingml/2006/main
     - https?://www\.verisign\.com/(rpa0|rpa|cps0)
@@ -247,9 +245,9 @@ regex:
     - ((api|www|platform)\.)?bing\.com$
   # Azure
     - md-ssd-.+\.blob\.core\.windows\.net$
-    - .+\.table\.core\.windows\.net
-    - .+\.blob\.core\.windows\.net
-    - .+\.opinsights\.azure\.com
+    - .+\.table\.core\.windows\.net$
+    - .+\.blob\.core\.windows\.net$
+    - .+\.opinsights\.azure\.com$
     - agentserviceapi\.azure-automation\.net$
     - agentserviceapi\.guestconfiguration\.azure\.com$
     - .+\.blob\.storage\.azure\.net$

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2}/assemblyline/filestore/transport/sftp.py

@@ -4,11 +4,7 @@ import posixpath
 import tempfile
 import warnings
 
-# Stop Blowfish deprecation warning
-with warnings.catch_warnings():
-    warnings.simplefilter("ignore")
-
-    import pysftp
+import sftpretty
 
 from io import BytesIO
 
@@ -30,7 +26,7 @@ def reconnect_retry_on_fail(func):
 
             try:
                 if not self.sftp:
-                    self.sftp = pysftp.Connection(self.host,
+                    self.sftp = sftpretty.Connection(self.host,
                                                   username=self.user,
                                                   password=self.password,
                                                   private_key=self.private_key,
@@ -47,7 +43,7 @@ def reconnect_retry_on_fail(func):
                 self.sftp.close()   # Just best effort.
 
             # The original func will reconnect automatically.
-            self.sftp = pysftp.Connection(self.host,
+            self.sftp = sftpretty.Connection(self.host,
                                           username=self.user,
                                           password=self.password,
                                           private_key=self.private_key,
@@ -80,13 +76,12 @@ class TransportSFTP(Transport):
         self.private_key = private_key
         self.private_key_pass = private_key_pass
         if not validate_host:
-            self.cnopts = pysftp.CnOpts()
-            self.cnopts.hostkeys = None
+            self.cnopts = sftpretty.CnOpts(knownhosts=None)
         else:
             self.cnopts = None
 
         # Connect on create
-        self.sftp = pysftp.Connection(self.host,
+        self.sftp = sftpretty.Connection(self.host,
                                       username=self.user,
                                       password=self.password,
                                       private_key=self.private_key,
@@ -131,7 +126,7 @@ class TransportSFTP(Transport):
     @reconnect_retry_on_fail
     def makedirs(self, path):
         path = self.normalize(path)
-        self.sftp.makedirs(path)
+        self.sftp.mkdir_p(path)
 
     # File based functions
     @reconnect_retry_on_fail

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.8.1.dev0
+Version: 4.8.1.dev2
 Summary: Assemblyline 4 - Automated malware analysis framework
 Author-email: CCCS Assemblyline development team <assemblyline@cyber.gc.ca>
 License-Expression: MIT
@@ -17,12 +17,10 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENCE.md
 Requires-Dist: arrow
-Requires-Dist: aiohttp
 Requires-Dist: lark
-Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
-Requires-Dist: pysftp
+Requires-Dist: sftpretty
 Requires-Dist: redis
 Requires-Dist: requests[socks]
 Requires-Dist: elasticsearch<9.0.0,>=8.0.0
@@ -35,7 +33,6 @@ Requires-Dist: assemblyline-toolbox>=0.4.0
 Requires-Dist: python-magic
 Requires-Dist: pytz
 Requires-Dist: apscheduler
-Requires-Dist: websocket_client<1.0.0
 Requires-Dist: elastic-apm[flask]>=6.13.0
 Requires-Dist: notifications-python-client
 Requires-Dist: rstr
@@ -46,13 +43,13 @@ Requires-Dist: chardet<6
 Requires-Dist: yara-python
 Requires-Dist: hauntedhouse==0.1.10
 Requires-Dist: magika
-Requires-Dist: paramiko<4
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: retrying; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: pyftpdlib; extra == "test"
-Requires-Dist: pyopenssl==23.3.0; extra == "test"
+Requires-Dist: pyopenssl; extra == "test"
+Requires-Dist: cryptography; extra == "test"
 Dynamic: license-file
 
 [![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu)

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2}/assemblyline.egg-info/requires.txt

@@ -1,10 +1,8 @@
 arrow
-aiohttp
 lark
-urllib3>=2.6.0
 python-baseconv
 boto3
-pysftp
+sftpretty
 redis
 requests[socks]
 elasticsearch<9.0.0,>=8.0.0
@@ -17,7 +15,6 @@ assemblyline-toolbox>=0.4.0
 python-magic
 pytz
 apscheduler
-websocket_client<1.0.0
 elastic-apm[flask]>=6.13.0
 notifications-python-client
 rstr
@@ -28,11 +25,11 @@ chardet<6
 yara-python
 hauntedhouse==0.1.10
 magika
-paramiko<4
 
 [test]
 pytest
 retrying
 pytest-mock
 pyftpdlib
-pyopenssl==23.3.0
+pyopenssl
+cryptography

{assemblyline-4.8.1.dev0 → assemblyline-4.8.1.dev2}/pyproject.toml

@@ -36,12 +36,10 @@ classifiers = [
 ]
 dependencies = [
     "arrow",
-    "aiohttp",
     "lark",
-    "urllib3>=2.6.0",
     "python-baseconv",
     "boto3",
-    "pysftp",
+    "sftpretty",
     "redis",
     "requests[socks]",
     "elasticsearch>=8.0.0,<9.0.0",
@@ -54,7 +52,6 @@ dependencies = [
     "python-magic",
     "pytz",
     "apscheduler",
-    "websocket_client<1.0.0",
     "elastic-apm[flask]>=6.13.0",
     "notifications-python-client",
     "rstr",
@@ -66,7 +63,6 @@ dependencies = [
     "yara-python",
     "hauntedhouse==0.1.10",
     "magika",
-    "paramiko<4",
 ]
 
 [project.urls]
@@ -78,7 +74,8 @@ test = [
     "retrying",
     "pytest-mock",
     "pyftpdlib",
-    "pyopenssl==23.3.0",
+    "pyopenssl",
+    "cryptography",
 ]
 
 [project.scripts]

assemblyline-4.8.1.dev0/assemblyline/VERSION

@@ -1 +0,0 @@
-4.8.1.dev0