misp-modules 2.4.196__py3-none-any.whl

misp_modules/lib/_vmray/rest_api.py

@@ -0,0 +1,148 @@
+#!/usr/bin/python3
+"""Python client library for VMRay REST API"""
+
+import base64
+import datetime
+import os.path
+import requests
+import urllib.parse
+
+# disable nasty certification warning
+# pylint: disable=no-member
+try:
+    requests.packages.urllib3.disable_warnings()
+except AttributeError:
+    try:
+        import urllib3
+        try:
+            urllib3.disable_warnings()
+        except AttributeError:
+            pass
+    except ImportError:
+        pass
+
+# pylint: disable=
+
+
+class VMRayRESTAPIError(Exception):
+    """Exception class that is used when API returns an error"""
+
+    def __init__(self, *args, **kwargs):
+        self.status_code = kwargs.pop("status_code", None)
+        Exception.__init__(self, *args, **kwargs)
+
+
+def handle_rest_api_result(result):
+    """Handle result of API request (check for errors)"""
+
+    if (result.status_code < 200) or (result.status_code > 299):
+        try:
+            json_result = result.json()
+        except ValueError:
+            raise VMRayRESTAPIError("API returned error %u: %s" % (result.status_code, result.text), status_code=result.status_code)
+
+        raise VMRayRESTAPIError(json_result.get("error_msg", "Unknown error"), status_code=result.status_code)
+
+
+class VMRayRESTAPI(object):
+    """VMRay REST API class"""
+
+    def __init__(self, server, api_key, verify_cert=True):
+        # split server URL into components
+        url_desc = urllib.parse.urlsplit(server)
+
+        # assume HTTPS if no scheme is specified
+        if url_desc.scheme == "":
+            server = "https://" + server
+
+        # save variables
+        self.server = server
+        self.api_key = api_key
+        self.verify_cert = verify_cert
+
+    def call(self, http_method, api_path, params=None, raw_data=False):
+        """Call VMRay REST API"""
+
+        # get function of requests package
+        requests_func = getattr(requests, http_method.lower())
+
+        # parse parameters
+        req_params = {}
+        file_params = {}
+
+        if params is not None:
+            for key, value in params.items():
+                if isinstance(value, (datetime.date,
+                                      datetime.datetime,
+                                      float,
+                                      int)):
+                    req_params[key] = str(value)
+                elif isinstance(value, str):
+                    req_params[key] = str(value)
+                elif isinstance(value, dict):
+                    filename = value["filename"]
+                    sample = value["data"]
+                    file_params[key] = (filename, sample, "application/octet-stream")
+                elif hasattr(value, "read"):
+                    filename = os.path.split(value.name)[1]
+                    # For the following block refer to DEV-1820
+                    try:
+                        filename.decode("ASCII")
+                    except (UnicodeDecodeError, UnicodeEncodeError):
+                        b64_key = key + "name_b64enc"
+                        byte_value = filename.encode("utf-8")
+                        b64_value = base64.b64encode(byte_value)
+
+                        filename = "@param=%s" % b64_key
+                        req_params[b64_key] = b64_value
+                    file_params[key] = (filename, value, "application/octet-stream")
+                else:
+                    raise VMRayRESTAPIError("Parameter \"%s\" has unknown type \"%s\"" % (key, type(value)))
+
+        # construct request
+        if file_params:
+            files = file_params
+        else:
+            files = None
+
+        # we need to adjust some stuff for POST requests
+        if http_method.lower() == "post":
+            req_data = req_params
+            req_params = None
+        else:
+            req_data = None
+
+        # do request
+        result = requests_func(self.server + api_path, data=req_data, params=req_params, headers={"Authorization": "api_key " + self.api_key}, files=files, verify=self.verify_cert, stream=raw_data)
+        handle_rest_api_result(result)
+
+        if raw_data:
+            return result.raw
+
+        # parse result
+        try:
+            json_result = result.json()
+        except ValueError:
+            raise ValueError("API returned invalid JSON: %s" % (result.text))
+
+        # if there are no cached elements then return the data
+        if "continuation_id" not in json_result:
+            return json_result.get("data", None)
+
+        data = json_result["data"]
+
+        # get cached results
+        while "continuation_id" in json_result:
+            # send request to server
+            result = requests.get("%s/rest/continuation/%u" % (self.server, json_result["continuation_id"]), headers={"Authorization": "api_key " + self.api_key}, verify=self.verify_cert)
+            handle_rest_api_result(result)
+
+            # parse result
+            try:
+                json_result = result.json()
+            except ValueError:
+                raise ValueError("API returned invalid JSON: %s" % (result.text))
+
+            data.extend(json_result["data"])
+
+        return data

misp_modules/lib/cof2misp/LICENSE-2.0.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

misp_modules/lib/cof2misp/cof.py

@@ -0,0 +1,165 @@
+"""
+Common Output Format for passive DNS library.
+
+Copyright 2021: Farsight Security (https://www.farsightsecurity.com/)
+
+Author: Aaron Kaplan <aaron@lo-res.org>
+
+Released under the Apache 2.0 license.
+See: https://www.apache.org/licenses/LICENSE-2.0.txt
+
+"""
+
+import ipaddress
+import sys
+import ndjson
+
+
+def is_valid_ip(ip: str) -> bool:
+    """Check if an IP address given as string would be convertible to
+    an ipaddress object (and thus if it is a valid IP).
+
+    Returns
+    --------
+    True on success, False on validation failure.
+    """
+
+    try:
+        ipaddress.ip_address(ip)
+    except Exception as ex:
+        print("is_valid_ip(%s) returned False. Reason: %s" % (ip, str(ex)), file = sys.stderr)
+        return False
+    return True
+
+
+def is_cof_valid_strict(d: dict) -> bool:
+    """Check the COF - do the full JSON schema validation.
+
+    Returns
+    --------
+    True on success, False on validation failure.
+    """
+    return True  # FIXME
+
+
+def is_cof_valid_simple(d: dict) -> bool:
+    """Check MANDATORY fields according to COF - simple check, do not do the full JSON schema validation.
+
+    Returns
+    --------
+    True on success, False on validation failure.
+    """
+
+    if "rrname" not in d:
+        print("Missing MANDATORY field 'rrname'", file = sys.stderr)
+        return False
+    if not isinstance(d['rrname'], str):
+        print("Type error: 'rrname' is not a JSON string", file = sys.stderr)
+        return False
+    if "rrtype" not in d:
+        print("Missing MANDATORY field 'rrtype'", file = sys.stderr)
+        return False
+    if not isinstance(d['rrtype'], str):
+        print("Type error: 'rrtype' is not a JSON string", file = sys.stderr)
+        return False
+    if "rdata" not in d:
+        print("Missing MANDATORY field 'rdata'", file = sys.stderr)
+        return False
+    if "rdata" not in d:
+        print("Missing MANDATORY field 'rdata'", file = sys.stderr)
+        return False
+    if not isinstance(d['rdata'], str) and not isinstance(d['rdata'], list):
+        print("'rdata' is not a list and not a string.", file = sys.stderr)
+        return False
+    if not ("time_first" in d and "time_last" in d) or ("zone_time_first" in d and "zone_time_last" in d):
+        print("We are missing EITHER ('first_seen' and 'last_seen') OR ('zone_time_first' and zone_time_last') fields",
+              file = sys.stderr)
+        return False
+    # currently we don't check the OPTIONAL fields. Sorry... to be done later.
+    return True
+
+
+def validate_cof(d: dict, strict=True) -> bool:
+    """Validate an input passive DNS COF (given as dict).
+    strict might be set to False in order to loosen the checking.
+    With strict==True, a full JSON Schema validation will happen.
+
+
+    Returns
+    --------
+    True on success, False on validation failure.
+    """
+    if not strict:
+        return is_cof_valid_simple(d)
+    else:
+        return is_cof_valid_strict(d)
+
+
+def validate_dnsdbflex(d: dict, strict=True) -> bool:
+    """
+    Validate if dict d is valid dnsdbflex. It should looks like this:
+    { "rrtype": <str>, "rrname": <str> }
+    """
+    if "rrname" not in d:
+        print("Missing MANDATORY field 'rrname'", file = sys.stderr)
+        return False
+    if not isinstance(d['rrname'], str):
+        print("Type error: 'rrname' is not a JSON string", file = sys.stderr)
+        return False
+    if "rrtype" not in d:
+        print("Missing MANDATORY field 'rrtype'", file = sys.stderr)
+        return False
+    if not isinstance(d['rrtype'], str):
+        print("Type error: 'rrtype' is not a JSON string", file = sys.stderr)
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    # simple, poor man's unit tests.
+
+    print(80 * "=", file = sys.stderr)
+    print("Unit Tests:", file = sys.stderr)
+    assert not is_valid_ip("a.2.3.4")
+    assert is_valid_ip("99.88.77.6")
+    assert is_valid_ip("2a0c:88:77:6::1")
+
+    # COF validation
+    print(80 * "=", file = sys.stderr)
+    print("COF unit tests....", file = sys.stderr)
+
+    mock_input = """{"count":1909,"rdata":["cpa.circl.lu"],"rrname":"www.circl.lu","rrtype":"CNAME","time_first":"1315586409","time_last":"1449566799"}
+{"count":2560,"rdata":["cpab.circl.lu"],"rrname":"www.circl.lu","rrtype":"CNAME","time_first":"1449584660","time_last":"1617676151"}"""
+
+    i = 0
+    for entry in ndjson.loads(mock_input):
+        retval = validate_cof(entry, strict = False)
+        assert retval
+        print("line %d is valid: %s" % (i, retval))
+        i += 1
+
+    test2 = '{"count": 2, "time_first": 1619556027, "time_last": 1619556034, "rrname": "westernunion.com.ph.unblock-all.com.beta.opera-mini.net.", "rrtype": "A", "bailiwick": "beta.opera-mini.net.", "rdata": ["185.26.181.253"]}'
+    for entry in ndjson.loads(test2):
+        assert validate_cof(entry)
+
+    # dnsdbflex validation
+    print(80 * "=", file = sys.stderr)
+    print("dnsdbflex unit tests....", file = sys.stderr)
+
+    mock_input = """{"rrname":"labs.deep-insights.ai.","rrtype":"A"}
+{"rrname":"www.deep-insights.ca.","rrtype":"CNAME"}
+{"rrname":"mail.deep-insights.ca.","rrtype":"CNAME"}
+{"rrname":"cpanel.deep-insights.ca.","rrtype":"A"}
+{"rrname":"webdisk.deep-insights.ca.","rrtype":"A"}
+{"rrname":"webmail.deep-insights.ca.","rrtype":"A"}"""
+
+    i = 0
+    for entry in ndjson.loads(mock_input):
+        retval = validate_dnsdbflex(entry, strict = False)
+        assert retval
+        print("dnsdbflex line %d is valid: %s" % (i, retval))
+        i += 1
+
+
+    print(80 * "=", file = sys.stderr)
+    print("Unit Tests DONE", file = sys.stderr)

misp_modules/lib/joe_mapping.py

@@ -0,0 +1,114 @@
+arch_type_mapping = {
+    'ANDROID': 'parse_apk',
+    'LINUX': 'parse_elf',
+    'WINDOWS': 'parse_pe'
+}
+domain_object_mapping = {
+    '@ip': {'type': 'ip-dst', 'object_relation': 'ip'},
+    '@name': {'type': 'domain', 'object_relation': 'domain'}
+}
+dropped_file_mapping = {
+    '@entropy': {'type': 'float', 'object_relation': 'entropy'},
+    '@file': {'type': 'filename', 'object_relation': 'filename'},
+    '@size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
+    '@type': {'type': 'mime-type', 'object_relation': 'mimetype'}
+}
+dropped_hash_mapping = {
+    'MD5': 'md5',
+    'SHA': 'sha1',
+    'SHA-256': 'sha256',
+    'SHA-512': 'sha512'
+}
+elf_object_mapping = {
+    'epaddr': 'entrypoint-address',
+    'machine': 'arch',
+    'osabi': 'os_abi'
+}
+elf_section_flags_mapping = {
+    'A': 'ALLOC',
+    'I': 'INFO_LINK',
+    'M': 'MERGE',
+    'S': 'STRINGS',
+    'T': 'TLS',
+    'W': 'WRITE',
+    'X': 'EXECINSTR'
+}
+file_object_fields = (
+    'filename',
+    'md5',
+    'sha1',
+    'sha256',
+    'sha512',
+    'ssdeep'
+)
+file_object_mapping = {
+    'entropy': {'type': 'float', 'object_relation': 'entropy'},
+    'filesize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
+    'filetype': {'type': 'mime-type', 'object_relation': 'mimetype'}
+}
+file_references_mapping = {
+    'fileCreated': 'creates',
+    'fileDeleted': 'deletes',
+    'fileMoved': 'moves',
+    'fileRead': 'reads',
+    'fileWritten': 'writes'
+}
+network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport')
+network_connection_object_mapping = {
+    'srcip': {'type': 'ip-src', 'object_relation': 'ip-src'},
+    'dstip': {'type': 'ip-dst', 'object_relation': 'ip-dst'},
+    'srcport': {'type': 'port', 'object_relation': 'src-port'},
+    'dstport': {'type': 'port', 'object_relation': 'dst-port'}
+}
+pe_object_fields = {
+    'entrypoint': {'type': 'text', 'object_relation': 'entrypoint-address'},
+    'imphash': {'type': 'imphash', 'object_relation': 'imphash'}
+}
+pe_object_mapping = {
+    'CompanyName': 'company-name',
+    'FileDescription': 'file-description',
+    'FileVersion': 'file-version',
+    'InternalName': 'internal-filename',
+    'LegalCopyright': 'legal-copyright',
+    'OriginalFilename': 'original-filename',
+    'ProductName': 'product-filename',
+    'ProductVersion': 'product-version',
+    'Translation': 'lang-id'
+}
+pe_section_object_mapping = {
+    'characteristics': {'type': 'text', 'object_relation': 'characteristic'},
+    'entropy': {'type': 'float', 'object_relation': 'entropy'},
+    'name': {'type': 'text', 'object_relation': 'name'},
+    'rawaddr': {'type': 'hex', 'object_relation': 'offset'},
+    'rawsize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
+    'virtaddr': {'type': 'hex', 'object_relation': 'virtual_address'},
+    'virtsize': {'type': 'size-in-bytes', 'object_relation': 'virtual_size'}
+}
+process_object_fields = {
+    'cmdline': 'command-line',
+    'name': 'name',
+    'parentpid': 'parent-pid',
+    'pid': 'pid',
+    'path': 'current-directory'
+}
+protocols = {
+    'tcp': 4,
+    'udp': 4,
+    'icmp': 3,
+    'http': 7,
+    'https': 7,
+    'ftp': 7
+}
+registry_references_mapping = {
+    'keyValueCreated': 'creates',
+    'keyValueModified': 'modifies'
+}
+regkey_object_mapping = {
+    'name': {'type': 'text', 'object_relation': 'name'},
+    'newdata': {'type': 'text', 'object_relation': 'data'},
+    'path': {'type': 'regkey', 'object_relation': 'key'}
+}
+signerinfo_object_mapping = {
+    'sigissuer': {'type': 'text', 'object_relation': 'issuer'},
+    'version': {'type': 'text', 'object_relation': 'version'}
+}