assemblyline-v4-service 4.4.0.24__py3-none-any.whl → 4.4.0.26__py3-none-any.whl

assemblyline_v4_service/common/safelist_helper.py

@@ -1,73 +0,0 @@
-from re import compile, IGNORECASE, match, search
-from typing import Dict, List
-from urllib.parse import urlparse
-
-from assemblyline.odm.base import DOMAIN_REGEX, IP_REGEX
-
-URL_REGEX = compile(
-    r"(?:(?:(?:[A-Za-z]*:)?//)?(?:\S+(?::\S*)?@)?(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}"
-    r"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|(?:(?:[A-Za-z0-9\u00a1-\uffff][A-Za-z0-9\u00a1-\uffff_-]{0,62})"
-    r"?[A-Za-z0-9\u00a1-\uffff]\.)+(?:xn--)?(?:[A-Za-z0-9\u00a1-\uffff]{2,}\.?))(?::\d{2,5})?)(?:[/?#][^\s,\\\\]*)?")
-
-
-def is_tag_safelisted(
-        value: str, tags: List[str],
-        safelist: Dict[str, Dict[str, List[str]]],
-        substring: bool = False) -> bool:
-    """
-    This method determines if a given value has any safelisted components.
-    :param value: The value to be checked if it has been safelisted
-    :param tags: The tags which will be used for grabbing specific values from the safelist
-    :param safelist: The safelist containing matches and regexs. The
-                     product of a service using self.get_api_interface().get_safelist().
-    :param substring: A flag that indicates if we should check if the value is contained within the match
-    :return: A boolean indicating if the value has been safelisted
-    """
-    if not value or not tags or not safelist:
-        return False
-
-    if not any(key in safelist for key in ["match", "regex"]):
-        return False
-
-    safelist_matches = safelist.get("match", {})
-    safelist_regexes = safelist.get("regex", {})
-
-    for tag in tags:
-        if tag in safelist_matches:
-            for safelist_match in safelist_matches[tag]:
-                if value.lower() == safelist_match.lower():
-                    return True
-                elif substring and safelist_match.lower() in value.lower():
-                    return True
-
-        if tag in safelist_regexes:
-            for safelist_regex in safelist_regexes[tag]:
-                if match(safelist_regex, value, IGNORECASE):
-                    return True
-
-    return False
-
-
-def contains_safelisted_value(val: str, safelist: Dict[str, Dict[str, List[str]]]) -> bool:
-    """
-    This method checks if a given value is part of a safelist
-    :param val: The given value
-    :param safelist: A dictionary containing matches and regexes for use in safelisting values
-    :return: A boolean representing if the given value is part of a safelist
-    """
-    if not val or not isinstance(val, str):
-        return False
-    ip = search(IP_REGEX, val)
-    url = search(URL_REGEX, val)
-    domain = search(DOMAIN_REGEX, val)
-    if ip is not None:
-        ip = ip.group()
-        return is_tag_safelisted(ip, ["network.dynamic.ip"], safelist)
-    elif domain is not None:
-        domain = domain.group()
-        return is_tag_safelisted(domain, ["network.dynamic.domain"], safelist)
-    elif url is not None:
-        url_pieces = urlparse(url.group())
-        domain = url_pieces.netloc
-        return is_tag_safelisted(domain, ["network.dynamic.domain"], safelist)
-    return False

assemblyline_v4_service/common/section_reducer.py

@@ -1,43 +0,0 @@
-from assemblyline_v4_service.common.result import Result, ResultSection
-from assemblyline_v4_service.common.tag_reducer import REDUCE_MAP
-
-
-def reduce(al_result: Result) -> Result:
-    """
-    This function goes through a result section recursively and try reduce the amount of
-    produced tags based on a reducer set for each specific tags
-
-    :param al_result: An Assemblyline result object
-    :return: Reduced Assemblyline result object
-    """
-    for section in al_result.sections:
-        _section_traverser(section)
-    return al_result
-
-
-def _section_traverser(section: ResultSection = None) -> ResultSection:
-    """
-    This function goes through each section and sends the tags to a function
-    that will reduce specific tags
-
-    :param section: An Assemblyline result section
-    :return: Reduced Assemblyline result section
-    """
-    for subsection in section.subsections:
-        _section_traverser(subsection)
-    if section.tags:
-        section.set_tags(_reduce_specific_tags(section.tags))
-    return section
-
-
-def _reduce_specific_tags(tags=None) -> dict:
-    """
-    This function is very much a work in progress. Currently the only tags that we
-    feel the need to reduce are unique uris and uri paths
-    :param tags: Dictionary of tag types and their values
-    :return: Dictionary of tag types and their reduced values
-    """
-    if tags is None:
-        tags = {}
-
-    return {tag_type: REDUCE_MAP.get(tag_type, lambda x: x)(tag_values) for tag_type, tag_values in tags.items()}

assemblyline_v4_service/common/tag_helper.py

@@ -1,117 +0,0 @@
-from re import match, search
-from typing import Any, Dict, List, Optional, Union
-
-from assemblyline.common.net import is_valid_domain, is_valid_ip
-from assemblyline.common.str_utils import safe_str
-from assemblyline.odm.base import DOMAIN_ONLY_REGEX, DOMAIN_REGEX, FULL_URI, IP_REGEX, URI_PATH
-from assemblyline_v4_service.common.result import ResultSection
-from assemblyline_v4_service.common.safelist_helper import is_tag_safelisted
-
-
-def add_tag(
-    result_section: ResultSection,
-    tag: str, value: Union[Any, List[Any]],
-    safelist: Dict[str, Dict[str, List[str]]] = None
-) -> bool:
-    """
-    This method adds the value(s) as a tag to the ResultSection. Can take a list of values or a single value.
-    :param result_section: The ResultSection that the tag will be added to
-    :param tag: The tag type that the value will be tagged under
-    :param value: The value, a single item or a list, that will be tagged under the tag type
-    :param safelist: The safelist containing matches and regexs. The product of a
-                     service using self.get_api_interface().get_safelist().
-    :return: Tag was successfully added
-    """
-    if safelist is None:
-        safelist = {}
-
-    tags_were_added = False
-    if not value:
-        return tags_were_added
-
-    if type(value) == list:
-        for item in value:
-            # If one tag is added, then return True
-            tags_were_added = _validate_tag(result_section, tag, item, safelist) or tags_were_added
-    else:
-        tags_were_added = _validate_tag(result_section, tag, value, safelist)
-    return tags_were_added
-
-
-def _get_regex_for_tag(tag: str) -> str:
-    """
-    This method returns a regular expression used for validating a certain tag type
-    :param tag: The type of tag
-    :return: The relevant regular expression
-    """
-    reg_to_match: Optional[str] = None
-    if "domain" in tag:
-        reg_to_match = DOMAIN_ONLY_REGEX
-    elif "uri_path" in tag:
-        reg_to_match = URI_PATH
-    elif "uri" in tag:
-        reg_to_match = FULL_URI
-    elif "ip" in tag:
-        reg_to_match = IP_REGEX
-    return reg_to_match
-
-
-def _validate_tag(
-    result_section: ResultSection,
-    tag: str,
-    value: Any,
-    safelist: Dict[str, Dict[str, List[str]]] = None
-) -> bool:
-    """
-    This method validates the value relative to the tag type before adding the value as a tag to the ResultSection.
-    :param result_section: The ResultSection that the tag will be added to
-    :param tag: The tag type that the value will be tagged under
-    :param value: The item that will be tagged under the tag type
-    :param safelist: The safelist containing matches and regexs. The product of a
-                     service using self.get_api_interface().get_safelist().
-    :return: Tag was successfully added
-    """
-    if safelist is None:
-        safelist = {}
-
-    if tag.startswith("network.static."):
-        network_tag_type = "static"
-    else:
-        network_tag_type = "dynamic"
-
-    regex = _get_regex_for_tag(tag)
-    if regex and not match(regex, value):
-        return False
-
-    if "ip" in tag and not is_valid_ip(value):
-        return False
-
-    if "domain" in tag and not is_valid_domain(value):
-            return False
-
-    if is_tag_safelisted(value, [tag], safelist):
-        return False
-
-    # if "uri" is in the tag, let's try to extract its domain/ip and tag it.
-    if "uri_path" not in tag and "uri" in tag:
-        # First try to get the domain
-        valid_domain = False
-        domain = search(DOMAIN_REGEX, value)
-        if domain:
-            domain = domain.group()
-            valid_domain = _validate_tag(result_section, f"network.{network_tag_type}.domain", domain, safelist)
-        # Then try to get the IP
-        valid_ip = False
-        ip = search(IP_REGEX, value)
-        if ip:
-            ip = ip.group()
-            valid_ip = _validate_tag(result_section, f"network.{network_tag_type}.ip", ip, safelist)
-
-        if value not in [domain, ip] and (valid_domain or valid_ip):
-            result_section.add_tag(tag, safe_str(value))
-        else:
-            return False
-    else:
-        result_section.add_tag(tag, safe_str(value))
-
-    return True

assemblyline_v4_service/common/tag_reducer.py

@@ -1,242 +0,0 @@
-import regex as re
-import os.path
-
-from copy import deepcopy
-from typing import List
-from urllib.parse import urlparse, parse_qs, urlunparse, urlencode, unquote
-
-NUMBER_REGEX = re.compile("[0-9]*")
-ALPHA_REGEX = re.compile("[a-zA-Z]*")
-ALPHANUM_REGEX = re.compile("[a-zA-Z0-9]*")
-BASE64_REGEX = re.compile("(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?")
-DO_NOT_REDUCE = ["netloc", "hostname"]
-
-
-def reduce_uri_tags(uris=None) -> List[str]:
-    """
-    The purpose of this helper function is to reduce the amount of unique uris to be tagged.
-    ex. If a sample makes a hundred network calls to four unqiue domains, with only one parameter
-    changing in the HTTP request each time, this should be synthesized to four uris to
-    be tagged, but with a placeholder for the parameter(s) that changes in each callout.
-    """
-    if uris is None:
-        uris = []
-
-    parsed_uris = []
-    reduced_uris = set()
-    for uri in uris:
-        parsed_uri = urlparse(uri)
-        # Match items we care about into a nice dictionary
-        uri_dict = {
-            "scheme": parsed_uri.scheme,      # scheme param
-            "netloc": parsed_uri.netloc,      # ""
-            "path": parsed_uri.path,          # ""
-            "params": parsed_uri.params,      # ""
-            "query": parsed_uri.query,        # ""
-            "fragment": parsed_uri.fragment,  # ""
-            "username": parsed_uri.username,  # None
-            "password": parsed_uri.password,  # None
-            "hostname": parsed_uri.hostname,  # None
-            "port": parsed_uri.port           # None
-        }
-
-        # We need to parse a couple of the returned params from urlparse more in-depth
-        if uri_dict["query"] != "":
-            # note that values of keys in dict will be in lists of length 1, which we don't want
-            uri_dict["query"] = parse_qs(uri_dict["query"])
-        if uri_dict["path"] != "":
-            # converting tuple to list
-            uri_dict["path"] = list(os.path.split(uri_dict["path"]))
-            # removing lone slashes
-            uri_dict["path"] = [not_slash for not_slash in uri_dict["path"] if not_slash != "/"]
-
-        parsed_uris.append(uri_dict)
-
-    # iterate through, comparing two parsed uris. if the percentage of similarity
-    # is greater than x, then they are sufficiently similar and can have parts
-    # replaced.
-
-    # time for the smarts
-    comparison_uris = deepcopy(parsed_uris)
-    for parsed_uri in parsed_uris:
-        # this flag will be used to check if this uri matches any other uri ever
-        totally_unique = True
-        for comparison_uri in comparison_uris:
-            if parsed_uri == comparison_uri:
-                continue
-            equal_keys = 0
-            total_list_len = 0
-            total_dict_len = 0
-            difference = {}
-            # now go through each key, and check for equality
-            for key in parsed_uri.keys():
-                val = parsed_uri[key]
-                comp_val = comparison_uri[key]
-
-                # if equal, add to count of similar keys
-                if type(val) == list:
-                    val_len = len(val)
-                    if val == comp_val:
-                        equal_keys += val_len
-                    else:
-                        difference[key] = dict()
-                        comp_len = len(comp_val)
-                        max_list_len = max(val_len, comp_len)
-                        for item in range(max_list_len):
-                            if item >= comp_len or item >= val_len:
-                                # bail!
-                                break
-                            if val[item] == comp_val[item]:
-                                equal_keys += 1
-                            else:
-                                difference[key][item] = []
-                                difference[key][item].append(val[item])
-                                difference[key][item].append(comp_val[item])
-                        total_list_len += val_len
-
-                elif type(val) == dict:
-                    val_len = len(val)
-                    if val == comp_val:
-                        equal_keys += val_len
-                    else:
-                        difference[key] = dict()
-                        if comp_val != "":
-                            comp_keys = list(comp_val.keys())
-                            val_keys = list(val.keys())
-                            all_keys = set(comp_keys + val_keys)
-                            val_len = len(all_keys)
-
-                            for item in all_keys:
-                                if val.get(item) and comp_val.get(item) and val[item] == comp_val[item]:
-                                    equal_keys += 1
-                                else:
-                                    difference[key][item] = []
-                                    if val.get(item):
-                                        difference[key][item].append(val[item])
-                                    if comp_val.get(item):
-                                        difference[key][item].append(comp_val[item])
-                        total_dict_len += val_len
-                else:  # Not dict or a list
-                    if val == comp_val:
-                        equal_keys += 1
-                    else:
-                        difference[key] = []
-                        difference[key].append(val)
-                        difference[key].append(comp_val)
-            # now find percentage similar
-            if total_dict_len > 1 and total_list_len > 1:
-                percentage_equal = equal_keys / (len(parsed_uri.keys()) - 2 + total_list_len + total_dict_len)
-            elif total_dict_len > 1 or total_list_len > 1:
-                percentage_equal = equal_keys / (len(parsed_uri.keys()) - 1 + total_list_len + total_dict_len)
-            else:
-                percentage_equal = equal_keys / (len(parsed_uri.keys()) + total_list_len + total_dict_len)
-
-            # if percentage equal is > some value (say 90), then we can say that
-            # urls are similar enough to reduce
-            if percentage_equal >= 0.80:
-                # So that we don't overwrite details
-                comparison_uri_copy = deepcopy(comparison_uri)
-                # somehow recognize where parameters are that match and replace them.
-                for item in difference.keys():
-                    # We don't want to replace the following:
-                    if item in DO_NOT_REDUCE:
-                        continue
-
-                    val = difference[item]
-                    if item == "query":
-                        for key in val.keys():
-                            placeholders = []
-                            # since each of these items is a list of lists
-                            for val_item in val[key]:
-                                # use regex to determine the parameter type
-                                value = val_item[0]
-                                placeholder = _get_placeholder(value)
-                                placeholders.append(placeholder)
-                            if len(set(placeholders)) == 1:
-                                # the same placeholder type is consistent with all values
-                                # update the url_dict value
-                                comparison_uri_copy[item][key] = list(set(placeholders))
-                            else:
-                                # the placeholder types vary
-                                comparison_uri_copy[item][key] = ",".join(placeholders)
-                    elif item == "path":
-                        placeholders = {}
-                        for key in val.keys():
-                            placeholders[key] = []
-                            for list_item in val[key]:
-                                # if / exists, pop the rest out
-                                if list_item != "/" and list_item[0] == "/":
-                                    # use regex to determine the parameter type
-                                    placeholder = _get_placeholder(list_item[1:])
-                                    placeholders[key].append("/"+placeholder)
-                                else:
-                                    placeholder = _get_placeholder(list_item)
-                                    placeholders[key].append(placeholder)
-                        for key in placeholders.keys():
-                            if len(set(placeholders[key])) == 1:
-                                # the same placeholder type is consistent with all values
-                                # update the comparison_uri_copy value
-                                comparison_uri_copy[item][key] = list(set(placeholders[key]))[0]
-                            else:
-                                # the placeholder types vary
-                                comparison_uri_copy[item][key] = ",".join(set(placeholders[key]))
-                    else:
-                        comparison_uri_copy[item] = _get_placeholder(val)
-
-                # now it's time to rejoin the parts of the url
-                reduced_uris.add(_turn_back_into_uri(comparison_uri_copy))
-                totally_unique = False
-
-        # Congratulations, you are one in a million
-        if totally_unique:
-            reduced_uris.add(_turn_back_into_uri(parsed_uri))
-    reduced_uris_list = list(reduced_uris)
-    # recursive_list = reduce_uri_tags(reduced_uris_list)
-    # if len(recursive_list) < len(reduced_uris_list):
-    #     return reduced_uris_list
-    # elif
-    # if reduce_uri_tags(reduced_uris_list))
-    return reduced_uris_list
-
-
-def _turn_back_into_uri(uri_parts: dict) -> str:
-    # turn the path back into a string
-    uri_parts["path"] = '/'.join(uri_parts["path"])
-    # turn the query back into a query string
-    # first, remove the list wrappers
-    if uri_parts["query"] != "":
-        for item in uri_parts["query"].keys():
-            uri_parts["query"][item] = uri_parts["query"][item][0]
-    uri_parts["query"] = unquote(urlencode(uri_parts["query"]))
-
-    uri_tuple = (uri_parts["scheme"], uri_parts["netloc"],
-                 uri_parts["path"], uri_parts["params"],
-                 uri_parts["query"], uri_parts["fragment"])
-    real_url = urlunparse(uri_tuple)
-    return real_url
-
-
-def _get_placeholder(val: str) -> str:
-    if not val:
-        return "${UNKNOWN_TYPE}"
-
-    if NUMBER_REGEX.fullmatch(val):
-        placeholder = "${NUMBER}"
-    elif ALPHA_REGEX.fullmatch(val):
-        placeholder = "${ALPHA}"
-    # Note that BASE64 Regex must happen before ALPHANUM regex or else ALPHANUM will hit on BASE64
-    elif BASE64_REGEX.fullmatch(val):
-        placeholder = "${BASE64}"
-    elif ALPHANUM_REGEX.fullmatch(val):
-        placeholder = "${ALPHA_NUM}"
-    else:
-        placeholder = "${UNKNOWN_TYPE}"
-    return placeholder
-
-
-REDUCE_MAP = {
-    "network.dynamic.uri": reduce_uri_tags,
-    "network.static.uri": reduce_uri_tags,
-    "network.dynamic.uri_path": reduce_uri_tags,
-    "network.static.uri_path": reduce_uri_tags
-}