malsearch 0.1.0__py3-none-any.whl

malsearch/VERSION.txt

@@ -0,0 +1 @@
+0.1.0

malsearch/__info__.py

@@ -0,0 +1,15 @@
+# -*- coding: UTF-8 -*-
+"""MalSearch package information.
+
+"""
+import os
+
+__author__    = "Alexandre D'Hondt"
+__copyright__ = "© 2024 A. D'Hondt"
+__email__     = "alexandre.dhondt@gmail.com"
+__license__   = "GPLv3 (https://www.gnu.org/licenses/gpl-3.0.fr.html)"
+__source__    = "https://github.com/packing-box/malsearch"
+
+with open(os.path.join(os.path.dirname(__file__), "VERSION.txt")) as f:
+    __version__ = f.read().strip()
+

malsearch/__init__.py

@@ -0,0 +1,90 @@
+# -*- coding: UTF-8 -*-
+from .clients import *
+from .clients import __all__ as _clients
+
+
+__all__ = ["download_sample", "download_samples"] + _clients
+
+_CLIENTS_MAP = {n.lower(): globals()[n] for n in _clients}
+
+
+def _valid_conf(path):
+    from configparser import ConfigParser
+    from os.path import exists, expanduser
+    path = expanduser(path)
+    if not exists(path):
+        raise ValueError("configuration file does not exist")
+    conf = ConfigParser()
+    try:
+        conf.read(path)
+        conf.path = path
+    except:
+        raise ValueError("invalid configuration file")
+    return conf
+
+
+def download_sample(hash, config=None, **kwargs):
+    import logging
+    logger = logging.getLogger("malsearch")
+    if config is None:
+        logger.error("no configuration file provided")
+        logger.info(f"you can create one at {config} manually (INI format with section 'API keys')")
+    else:
+        import datetime as dt
+        from os.path import exists, join
+        p = join(kwargs.get('output_dir', "."), hash)
+        if exists(p) and not kwargs.get('overwrite'):
+            logger.info(f"'{p}' already exists")
+            return
+        if isinstance(config, str):
+            config = _valid_conf(config)
+        clients = []
+        for n in config['API keys']:
+            if n in (kwargs.get('skip') or []):
+                logger.debug(f"{n} skipped")
+                continue
+            if config.has_section("Disabled"):
+                t = config['Disabled'].get(n)
+                if t is not None:
+                    try:
+                        if dt.datetime.strptime(t, "%d/%m/%Y %H:%M:%S") < dt.datetime.now():
+                            from contextlib import nullcontext
+                            with kwargs.get('lock') or nullcontext():
+                                config['Disabled'].pop(n)
+                                with open(config.path, 'w') as f:
+                                    config.write(f)
+                        else:
+                            logger.warning(f"{n} is disabled until {t}")
+                            continue
+                    except ValueError:
+                        logger.warning(f"{n} is disabled")
+                        continue
+            clients.append(n)
+        if len(clients) == 0:
+            logger.warning("no download client available/enabled")
+        logger.debug(f"clients: {', '.join(clients)}")
+        for n in clients:
+            logger.debug(f"trying {n}...")
+            cls = _CLIENTS_MAP[n]
+            if cls.__base__.__name__ == "API":
+                kwargs['api_key'] = config['API keys'].get(n)
+            client = cls(config=config, **kwargs)
+            try:
+                client.get_file_by_hash(hash)
+                if hasattr(client, "content") and client.content is not None and len(client.content) > 0:
+                    logger.debug("found sample !")
+                    return
+            except ValueError as e:
+                logger.debug(e)
+            except Exception as e:
+                logger.exception(e)
+    logger.warning(f"could not find the sample with hash {hash}")
+
+
+def download_samples(*hashes, max_workers=5, **kwargs):
+    from concurrent.futures import ThreadPoolExecutor as Pool
+    from threading import Lock
+    kwargs['lock'] = Lock()
+    with Pool(max_workers=max_workers) as executor:
+        for h in hashes:
+            executor.submit(download_sample, h.lower(), **kwargs)

malsearch/__main__.py

@@ -0,0 +1,60 @@
+# -*- coding: UTF-8 -*-
+from .__info__ import __author__, __copyright__, __email__, __license__, __source__, __version__
+
+
+def _parser(name, description, examples):
+    from argparse import ArgumentParser, RawTextHelpFormatter
+    descr = f"{name} {__version__}\n\nAuthor   : {__author__} ({__email__})\nCopyright: {__copyright__}\nLicense  :" \
+            f" {__license__}\nSource   : {__source__}\n\n{description}.\n\n"
+    return ArgumentParser(description=descr, formatter_class=RawTextHelpFormatter, add_help=False,
+                          epilog="usage examples:\n- " + "\n- ".join(examples) if len(examples) > 0 else None)
+
+
+def _setup(parser):
+    args = parser.parse_args()
+    if hasattr(args, "verbose"):
+        import logging
+        logging.basicConfig(level=[logging.INFO, logging.DEBUG][args.verbose])
+    if not args.disable_cache:
+        from requests_cache import install_cache
+        install_cache("malsearch",
+            use_cache_dir=True,                 # Save files in the default user cache dir
+            cache_control=True,                 # Use Cache-Control response headers for expiration, if available
+            expire_after=86400,                 # Otherwise expire responses after one day
+            allowable_codes=[404],              # Cache 400 responses as a solemn reminder of your failures
+            allowable_methods=['GET', 'POST'],  # Cache whatever HTTP methods you want
+            ignored_parameters=['api_key'],     # Don't match this request param, and redact if from the cache
+            match_headers=['Accept-Language'],  # Cache a different response per language
+            stale_if_error=True,                # In case of request errors, use stale cache data if possible
+        )
+    return args
+
+
+def main():
+    from os import makedirs
+    from .__init__ import _valid_conf, download_samples
+    from .clients.__common__ import _valid_hash
+    parser = _parser("MalSearch", "This tool is aimed to search for malware samples across some public databases", [])
+    parser.add_argument("sample_hash", type=_valid_hash, nargs="*", help="input hash")
+    parser.add_argument("-f", "--from-file", help="get hashes from the target file (newline-separated list)")
+    opt = parser.add_argument_group("optional arguments")
+    opt.add_argument("-c", "--config", default="~/.malsearch.conf", type=_valid_conf, help="INI configuration file")
+    opt.add_argument("-o", "--output-dir", default=".", help="output directory for downloaded samples")
+    opt.add_argument("-s", "--skip", nargs="*", help="skip the specified clients while downloading samples")
+    opt.add_argument("-u", "--unpacked", action="store_true",
+                     help="if available and target sample is packed, download unpacked version too")
+    opt.add_argument("--disable-cache", action="store_true", help="disable requests cache")
+    extra = parser.add_argument_group("extra arguments")
+    extra.add_argument("-h", "--help", action="help", help="show this help message and exit")
+    extra.add_argument("-v", "--verbose", action="store_true", help="display debug information (default: False)")
+    args = _setup(parser)
+    if args.from_file is not None:
+        with open(args.from_file) as f:
+            for h in f.readlines():
+                args.sample_hash.append(_valid_hash(h.strip()))
+    makedirs(args.output_dir, exist_ok=True)
+    if len(args.sample_hash) > 0:
+        download_samples(*args.sample_hash, **vars(args))
+    else:
+        import logging
+        logging.getLogger("malsearch").warning("nothing to download")

malsearch/clients/__common__.py

@@ -0,0 +1,162 @@
+# -*- coding: UTF-8 -*-
+import logging
+
+
+__all__ = ["API", "Web"]
+
+_MIN_BACKOFF = 300
+
+logger = logging.getLogger("malsearch")
+
+
+def _valid_hash(hash, *types):
+    import re
+    types_map = {'md5': 32, 'sha1': 40, 56: 'sha224', 'sha256': 64, 'sha384': 96, 'sha512': 128}
+    pattern = "|".join(f"[0-9a-f]{{{types_map[t]}}}" for t in (types or types_map.keys()))
+    if re.match(r"{}".format(f"^{pattern}$"), hash, re.I) is None:
+        raise ValueError("hash type not supported")
+    return hash.lower()
+
+
+def hashtype(*types):
+    def _wrapper(f):
+        from functools import wraps
+        @wraps(f)
+        def _subwrapper(*args, **kwargs):
+            _valid_hash(args[0] if isinstance(args[0], str) else args[1], *types)
+            return f(*args, **kwargs)
+        return _subwrapper
+    return _wrapper
+
+
+class _Base:
+    def __init__(self, **kwargs):
+        self.name = self.__class__.__name__.lower()
+        for k, v in kwargs.items():
+            setattr(self, f"_{k}", v)
+    
+    def _decode(self, codec):
+        try:
+            import codext as codecs
+        except ImportError:
+            import codecs
+        try:
+            self.content = codecs.decode(self.content, codec)
+        except ValueError as e:
+            logger.debug(f"cannot decode ({e})")
+        except Exception as e:
+            logger.exception(e)
+        return self
+    
+    def _save(self, filename):
+        from os.path import join
+        try:
+            c = self.content
+            with open(join(self._output_dir, filename), 'wb') as f:
+                f.write(c)
+        except AttributeError:
+            logger.debug("cannot save (no content downloaded)")
+        except Exception as e:
+            logger.exception(e)
+        return self
+
+    def _unzip(self, password=None):
+        from io import BytesIO
+        try:
+            with BytesIO(self.content) as buffer:
+                from pyzipper import AESZipFile
+                from zipfile import ZipFile
+                for ZipClass in [ZipFile, AESZipFile]:
+                    try:
+                        with ZipClass(buffer) as zip_file:
+                            if password:
+                                zip_file.setpassword(password)
+                            with zip_file.open(zip_file.namelist()[0]) as file:
+                                self.content = file.read()
+                                logger.debug("unzipped with {ZipClass}")
+                                break
+                    except NotImplementedError:
+                        continue
+        except AttributeError:
+            logger.debug("cannot unzip (no content downloaded)")
+        except Exception as e:
+            logger.exception(e)
+        return self
+
+
+class API(_Base):
+    def __request(self, path, **kwargs):
+        from requests import exceptions, get, post
+        for i in ["headers", "params"]:
+            n = getattr(self, f"_api_key_{i.rstrip('s')}", None)
+            if n:
+                d = kwargs.pop(i, {}) or {}
+                d[n] = self._api_key
+                kwargs[i] = d
+        n = getattr(self, "_auth_method", None)
+        if n:
+            kwargs['headers'] = kwargs['headers'] or {}
+            kwargs['headers']['Authorization'] = f"{self._auth_method} {self._api_key}"
+        r = (get if 'params' in kwargs else post)(f"{self.url}/{path}", **kwargs)
+        try:
+            r.raise_for_status()
+            self.content = r.content
+        except exceptions.RequestException as e:
+            if r.status_code == 403:
+                import datetime as dt
+                from contextlib import nullcontext
+                with kwargs.get('lock') or nullcontext():
+                    if not self._config.has_section("Disabled"):
+                        self._config.add_section("Disabled")
+                    boff = dt.timedelta(seconds=getattr(self, "backoff", _MIN_BACKOFF))
+                    self._config['Disabled'][self.name] = dt.datetime.strftime(dt.datetime.now() + boff,
+                                                                               "%d/%m/%Y %H:%M:%S")
+                    with open(self._config.path, 'w') as f:
+                        self._config.write(f)
+            logger.exception(e)
+        try:
+            self.json = r.json()
+        except:
+            pass
+
+    def _get(self, path, params=None, headers=None):
+        """ Perform a GET request.
+        
+        :param path:    API endpoint (relative path).
+        :param params:  Dictionary of URL parameters.
+        :param headers: Optional dictionary of request headers.
+        :return:        JSON data or error message.
+        """
+        self.__request(path, params=params, headers=headers)
+        return self
+    
+    def _post(self, path, data=None, json=None, headers=None):
+        """ Perform a POST request.
+
+        :param path:    API endpoint (relative path).
+        :param data:    Dictionary or list of tuples for form-encoded data.
+        :param json:    Dictionary to send JSON data.
+        :param headers: Optional dictionary of request headers.
+        :return:        JSON data or error message.
+        """
+        self.__request(path, data=data, json=json, headers=headers)
+        return self
+
+
+class Web(_Base):
+    def _get(self, path, params=None, headers=None):
+        """ Perform a GET request.
+        
+        :param path:    API endpoint (relative path).
+        :param params:  Dictionary of URL parameters.
+        :param headers: Optional dictionary of request headers.
+        :return:        BeautifulSoup-parsed HTML object or None.
+        """
+        from bs4 import BeautifulSoup
+        from requests import exceptions, get
+        try:
+            r = get(path, params=params, headers=headers)
+            r.raise_for_status()
+            return BeautifulSoup(r.content, "html.parser")
+        except exceptions.RequestException as e:
+            return

malsearch/clients/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: UTF-8 -*-
+from .malpedia import Malpedia
+from .malshare import MalShare
+from .malwarebazaar import MalwareBazaar
+from .triage import Triage
+from .virusshare import VirusShare
+from .virustotal import VirusTotal
+
+
+__all__ = ["Malpedia", "MalShare", "MalwareBazaar", "Triage", "VirusShare", "VirusTotal"]
+

malsearch/clients/malpedia.py

@@ -0,0 +1,16 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import hashtype, API
+
+
+__all__ = ["Malpedia"]
+
+
+class Malpedia(API):
+    doc = "https://malpedia.caad.fkie.fraunhofer.de/usage/api"
+    url = "https://malpedia.caad.fkie.fraunhofer.de/api"
+    _auth_method = "APIToken"
+
+    @hashtype("md5", "sha256")
+    def get_file_by_hash(self, hash):
+        self._get(f"get/sample/{hash}/zip")._save(hash)
+        #TODO: parse ZIP for unpacked version

malsearch/clients/malshare.py

@@ -0,0 +1,15 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import hashtype, API
+
+
+__all__ = ["MalShare"]
+
+
+class MalShare(API):
+    doc = "https://malshare.com/doc.php"
+    url = "https://malshare.com"
+    _api_key_param = "api_key"
+
+    @hashtype("md5", "sha1", "sha256")
+    def get_file_by_hash(self, hash):
+        self._get("api.php", params={'action': "getfile", 'hash': hash})._save(hash)

malsearch/clients/malwarebazaar.py

@@ -0,0 +1,16 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import hashtype, API
+
+
+__all__ = ["MalwareBazaar"]
+
+
+class MalwareBazaar(API):
+    doc = "https://bazaar.abuse.ch/api"
+    url = "https://mb-api.abuse.ch/api/v1"
+    _api_key_header = "API-KEY"
+
+    @hashtype("sha256")
+    def get_file_by_hash(self, hash):
+        # daily limit: 2000/IP/day
+        self._post("", data={'query': "get_file", f'sha256_hash': hash})._unzip(b"infected")._save(hash)

malsearch/clients/triage.py

@@ -0,0 +1,21 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import hashtype, API
+
+
+__all__ = ["Triage"]
+
+
+class Triage(API):
+    doc = "https://tria.ge/docs"
+    url = "https://tria.ge/api/v0"
+    _auth_method = "Bearer"
+
+    @hashtype("md5", "sha1", "sha256", "sha512")
+    def get_file_by_hash(self, hash):
+        hashtype = {32: "md5", 40: "sha1", 64: "sha256", 128: "sha512"}[len(hash)]
+        self._get("search", params={'query': f"{hashtype}:{hash}"})
+        try:
+            sample_id = self.json['data'][0]['id']
+            self._get(f"samples/{sample_id}/sample")._save(hash)
+        except IndexError:
+            delattr(self, "content")

malsearch/clients/virusshare.py

@@ -0,0 +1,15 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import hashtype, API
+
+
+__all__ = ["VirusShare"]
+
+
+class VirusShare(API):
+    doc = "https://virusshare.com/apiv2_reference"
+    url = "https://virusshare.com/apiv2"
+    _api_key_param = "apikey"
+    
+    @hashtype("md5", "sha1", "sha224", "sha256", "sha384", "sha512")
+    def get_file_by_hash(self, hash):
+         self._get("download", params={'hash': hash})._save(hash)

malsearch/clients/virustotal.py

@@ -0,0 +1,21 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import hashtype, API
+
+
+__all__ = ["VirusTotal"]
+
+
+class VirusTotal(API):
+    doc = "https://docs.virustotal.com/reference/overview"
+    url = "https://www.virustotal.com/api/v3"
+    _api_key_header = "X-Apikey"
+
+    @hashtype("md5", "sha1", "sha256")
+    def get_file_by_hash(self, hash):
+        if self._unpacked:
+            data = self._get(f"/files/{hash}")
+            for n in data['data']['attributes']['names']:
+                if n.endswith("_unpacked"):
+                    hash2, _ = n.split("_", 1)
+                    self._get(f"/files/{hash2}/download")._save(hash2)
+        self._get(f"files/{hash}/download")._unzip(b"infected")._save(hash)