assemblyline-v4-service 4.4.0.24__py3-none-any.whl → 4.4.0.26__py3-none-any.whl

assemblyline_v4_service/common/extractor/__init__.py

@@ -1 +0,0 @@
-

assemblyline_v4_service/common/extractor/base64.py

@@ -1,86 +0,0 @@
-"""
-Base 64 encoded text
-"""
-
-import binascii
-import regex as re
-import warnings
-
-from typing import Dict, List, Tuple
-
-HTML_ESCAPE_RE = rb'&#(?:x[a-fA-F0-9]{1,4}|\d{1,4});'
-BASE64_RE = rb'(?:[A-Za-z0-9+/]{4,}(?:<\x00  \x00)?(?:&#13;|&#xD;)?(?:&#10;|&#xA)?\r?\n?){5,}' \
-            rb'[A-Za-z0-9+/]{2,}={0,2}'
-
-CAMEL_RE = rb'(?i)[a-z]+'
-HEX_RE = rb'(?i)[a-f0-9]+'
-MIN_B64_CHARS = 6
-
-
-def base64_search(text: bytes) -> Dict[bytes, bytes]:
-    """
-    Find all base64 encoded sections in a text.
-    Args:
-        text: The text to search.
-    Returns:
-        A dictionary with the original base64 encoded sections as keys
-        and the corresponding decoded data as values.
-    """
-    warnings.warn("base64_search is depricated, use find_base64 instead", DeprecationWarning)
-    b64_matches = {}
-    for b64_match in re.findall(BASE64_RE, text):
-        if b64_match in b64_matches:
-            continue
-        b64_string = re.sub(HTML_ESCAPE_RE, b'', b64_match).replace(b'\n', b'').replace(b'\r', b'') \
-            .replace(b'<\x00  \x00', b'')
-        if re.fullmatch(HEX_RE, b64_string):
-            # Hexadecimal characters are a subset of base64
-            # Hashes commonly are hex and have multiple of 4 lengths
-            continue
-        if re.fullmatch(CAMEL_RE, b64_string):
-            # Camel case text can be confused for base64
-            # It is common in scripts as names
-            continue
-        uniq_char = set(b64_string)
-        if len(uniq_char) > MIN_B64_CHARS and len(b64_string) % 4 == 0:
-            try:
-                b64_result = binascii.a2b_base64(b64_string)
-                b64_matches[b64_match] = b64_result
-            except binascii.Error:
-                pass
-    return b64_matches
-
-
-def find_base64(data: bytes) -> List[Tuple[bytes, int, int]]:
-    """
-    Find all base64 encoded sections in some data.
-
-    Args:
-        data: The data to search.
-    Returns:
-        A list of decoded base64 sections and the location indexes of the section
-        in the original data.
-    """
-    b64_matches = []
-    for b64_match in re.finditer(BASE64_RE, data):
-        b64_string = re.sub(HTML_ESCAPE_RE, b'', b64_match.group()).replace(b'\n', b'').replace(b'\r', b'') \
-            .replace(b'<\x00  \x00', b'')
-        if len(b64_string) % 4 != 0 or len(set(b64_string)) <= MIN_B64_CHARS:
-            continue
-        if re.fullmatch(HEX_RE, b64_string):
-            # Hexadecimal characters are a subset of base64
-            # Hashes commonly are hex and have multiple of 4 lengths
-            continue
-        if re.fullmatch(CAMEL_RE, b64_string):
-            # Camel case text can be confused for base64
-            # It is common in scripts as names
-            continue
-        if b64_string.count(b'/')/len(b64_string) > 3/32:
-            # If there are a lot of / it as more likely a path
-            continue
-        try:
-            b64_result = binascii.a2b_base64(b64_string)
-            b64_matches.append((b64_result, b64_match.start(), b64_match.end()))
-        except binascii.Error:
-            pass
-    return b64_matches

assemblyline_v4_service/common/extractor/pe_file.py

@@ -1,51 +0,0 @@
-import regex as re
-
-from typing import List, Tuple
-
-import pefile
-
-EXEDOS_RE = rb'(?s)This program cannot be run in DOS mode'
-EXEHEADER_RE = rb'(?s)MZ.{32,1024}PE\000\000'
-
-
-def _find_pe_files_with_offset(data: bytes) -> List[Tuple[bytes, int, int]]:
-    """
-    Searches for any PE files within data
-
-    Args:
-        data: The data to search
-    Returns:
-        A list tuples containing: The found PE file, the starting offset and the end offset
-    """
-    pe_files: List[Tuple[bytes, int, int]] = []
-    offset = 0
-    while offset < len(data):
-        match = re.search(EXEHEADER_RE, data)
-        if not match:
-            return pe_files
-        pe_data = data[offset:]
-        if not re.search(EXEDOS_RE, pe_data):
-            return pe_files
-        try:
-            pe = pefile.PE(data=pe_data)
-            size = max(section.PointerToRawData + section.SizeOfRawData for section in pe.sections)
-            if size == 0:
-                return pe_files
-            end = offset+size
-            pe_files.append((data[offset:end], offset, end))
-            offset = end
-        except Exception:
-            return pe_files
-    return pe_files
-
-
-def find_pe_files(data: bytes) -> List[bytes]:
-    """
-    Searches for any PE files within data
-
-    Args:
-        data: The data to search
-    Returns:
-        A list of found PE files
-    """
-    return [pe[0] for pe in _find_pe_files_with_offset(data)]

assemblyline_v4_service/common/icap.py

@@ -1,149 +0,0 @@
-import os
-import socket
-
-from assemblyline.common.str_utils import safe_str
-
-ICAP_OK = b'ICAP/1.0 200 OK'
-
-
-# noinspection PyBroadException
-class IcapClient(object):
-    """
-    A limited Internet Content Adaptation Protocol client.
-
-    Currently only supports RESPMOD as that is all that is required to interop
-    with most ICAP based AV servers.
-    """
-
-    RESP_CHUNK_SIZE = 65565
-    MAX_RETRY = 3
-
-    def __init__(self, host, port, respmod_service="av/respmod", action="", timeout=30, number_of_retries=MAX_RETRY):
-        self.host = host
-        self.port = port
-        self.service = respmod_service
-        self.action = action
-        self.socket = None
-        self.timeout = timeout
-        self.kill = False
-        self.number_of_retries = number_of_retries
-        self.successful_connection = False
-
-    def scan_data(self, data, name=None):
-        return self._do_respmod(name or 'filetoscan', data)
-
-    def scan_local_file(self, filepath):
-        filename = os.path.basename(filepath)
-        with open(filepath, 'r') as f:
-            data = f.read()
-            return self.scan_data(data, filename)
-
-    def options_respmod(self):
-        request = f"OPTIONS icap://{self.host}:{self.port}/{self.service} ICAP/1.0\r\n\r\n"
-
-        for i in range(self.number_of_retries):
-            if self.kill:
-                self.kill = False
-                return
-            try:
-                if not self.socket:
-                    self.socket = socket.create_connection((self.host, self.port), timeout=self.timeout)
-                    self.successful_connection = True
-                self.socket.sendall(request.encode())
-                response = temp_resp = self.socket.recv(self.RESP_CHUNK_SIZE)
-                while len(temp_resp) == self.RESP_CHUNK_SIZE:
-                    temp_resp = self.socket.recv(self.RESP_CHUNK_SIZE)
-                    response += temp_resp
-                if not response or not response.startswith(ICAP_OK):
-                    raise Exception(f"Unexpected OPTIONS response: {response}")
-                return response.decode()
-            except Exception:
-                self.successful_connection = False
-                try:
-                    self.socket.close()
-                except Exception:
-                    pass
-                self.socket = None
-                if i == (self.number_of_retries-1):
-                    raise
-
-        raise Exception("Icap server refused to respond.")
-
-    @staticmethod
-    def chunk_encode(data):
-        chunk_size = 8160
-        out = b""
-        offset = 0
-        while len(data) < offset * chunk_size:
-            out += "1FEO\r\n"
-            out += data[offset * chunk_size:(offset + 1) * chunk_size]
-            out += "\r\n"
-            offset += 1
-
-        out += b"%X\r\n" % len(data[offset * chunk_size:])
-        out += data[offset * chunk_size:]
-        out += b"\r\n0\r\n\r\n"
-
-        return out
-
-    def _do_respmod(self, filename, data):
-        encoded = self.chunk_encode(data)
-
-        # ICAP RESPMOD req-hdr is the start of the original HTTP request.
-        respmod_req_hdr = "GET /{FILENAME} HTTP/1.1\r\n\r\n".format(FILENAME=safe_str(filename))
-
-        # ICAP RESPMOD res-hdr is the start of the HTTP response for above request.
-        respmod_res_hdr = (
-            "HTTP/1.1 200 OK\r\n"
-            "Transfer-Encoding: chunked\r\n\r\n")
-
-        res_hdr_offset = len(respmod_req_hdr)
-        res_bdy_offset = len(respmod_res_hdr) + res_hdr_offset
-
-        # The ICAP RESPMOD header. Note:
-        # res-hdr offset should match the start of the GET request above.
-        # res-body offset should match the start of the response above.
-
-        respmod_icap_hdr = (
-            f"RESPMOD icap://{self.host}:{self.port}/{self.service}{self.action} ICAP/1.0\r\n"
-            f"Host: {self.host}:{self.port}\r\n"
-            "Allow: 204\r\n"
-            f"Encapsulated: req-hdr=0, res-hdr={res_hdr_offset}, res-body={res_bdy_offset}\r\n\r\n"
-        )
-
-        serialized_request = b"%s%s%s%s" % (respmod_icap_hdr.encode(), respmod_req_hdr.encode(),
-                                            respmod_res_hdr.encode(), encoded)
-
-        for i in range(self.number_of_retries):
-            if self.kill:
-                self.kill = False
-                return
-            try:
-                if not self.socket:
-                    self.socket = socket.create_connection((self.host, self.port), timeout=self.timeout)
-                    self.successful_connection = True
-                self.socket.sendall(serialized_request)
-                response = temp_resp = self.socket.recv(self.RESP_CHUNK_SIZE)
-                while len(temp_resp) == self.RESP_CHUNK_SIZE:
-                    temp_resp = self.socket.recv(self.RESP_CHUNK_SIZE)
-                    response += temp_resp
-
-                return response.decode()
-            except Exception:
-                self.successful_connection = False
-                try:
-                    self.socket.close()
-                except Exception:
-                    pass
-                self.socket = None
-                if i == (self.number_of_retries-1):
-                    raise
-
-        raise Exception("Icap server refused to respond.")
-
-    def close(self):
-        self.kill = True
-        try:
-            self.socket.close()
-        except Exception:
-            pass

assemblyline_v4_service/common/keytool_parse.py

@@ -1,66 +0,0 @@
-from re import split
-from subprocess import Popen, PIPE
-from assemblyline.common.str_utils import safe_str
-from typing import List
-
-
-class Certificate():
-    def __init__(self):
-        self.raw = ""
-        self.issuer = ""
-        self.owner = ""
-        self.country = ""
-        self.valid_from = ""
-        self.valid_to = ""
-
-
-def keytool_printcert(cert_path: str) -> None:
-    """
-    This function runs the 'keytool -printcert' command against a provided file
-
-    :param cert_path: A path to a certificate
-    :return: the string output of 'keytool -printcert' or None
-    """
-    stdout, _ = Popen(["keytool", "-printcert", "-file", cert_path],
-                      stderr=PIPE, stdout=PIPE).communicate()
-    stdout = safe_str(stdout)
-
-    if stdout and "keytool error" not in stdout:
-        return stdout
-
-    return None
-
-
-def certificate_chain_from_printcert(printcert: str) -> List[Certificate]:
-    """
-    This function parses the output of 'keytool -printcert' and creates a list of Certificate objects.
-    The input to this function is the output of keytool_printcert
-
-    :param printcert: the string output of 'keytool -printcert'
-    :return: a list of the parsed out certificates. If only one certificate
-             is present and not a chain, then the list will have one element.
-    """
-    certs: List[Certificate] = []
-
-    for cert_str in split(r'Certificate\[\d+\]:', printcert):  # split printcert output in case of certificate chain
-        if cert_str == '':
-            continue
-        cert = Certificate()
-        cert.raw = cert_str.strip()
-        for line in cert_str.splitlines():
-            if "Owner:" in line:
-                cert.owner = line.split(": ", 1)[1]
-                country = cert.owner.split("C=")
-                if len(country) != 1:
-                    cert.country = country[1]
-
-            elif "Issuer:" in line:
-                cert.issuer = line.split(": ", 1)[1]
-
-            elif "Valid from:" in line:
-                cert.valid_from = line.split(": ", 1)[1].split(" until:")[0]
-                cert.valid_to = line.rsplit(": ", 1)[1]
-
-        certs.append(cert)
-
-    return certs