supplier-meesho.webflow 1.0.0

package/MobSF/mobsf/MalwareAnalyzer/views/MalwareDomainCheck.py

@@ -0,0 +1,226 @@
+# -*- coding: utf_8 -*-
+# Module for Malware Analysis
+import io
+import logging
+import re
+from pathlib import Path
+from socket import (
+    gaierror,
+    gethostbyname,
+)
+from urllib.parse import urlparse
+
+from django.conf import settings
+
+import IP2Location
+
+from mobsf.MobSF.utils import (
+    is_internet_available,
+    update_local_db,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class MalwareDomainCheck:
+
+    def __init__(self):
+        self.sig_dir = Path(settings.SIGNATURE_DIR)
+        self.malwaredomainlist = self.sig_dir / 'malwaredomainlist'
+        self.maltrail = self.sig_dir / 'maltrail-malware-domains.txt'
+        self.iplocbin = self.sig_dir / 'IP2LOCATION-LITE-DB5.IPV6.BIN'
+        self.result = {}
+        self.domainlist = None
+        self.IP2Loc = IP2Location.IP2Location()
+
+    def update_malware_db(self):
+        """Check for update in malware DB."""
+        try:
+            mal_db = self.malwaredomainlist
+            resp = update_local_db('Malware', settings.MALWARE_DB_URL, mal_db)
+            if not resp:
+                return
+            # DB needs update
+            # Check2: DB Syntax Changed
+            line = resp.decode('utf-8', 'ignore').split('\n')[0]
+            lst = line.split('",')
+            if len(lst) == 10:
+                # DB Format is not changed. Let's update DB
+                logger.info('Updating Malware Database')
+                with open(mal_db, 'wb') as wfp:
+                    wfp.write(resp)
+            else:
+                logger.warning('Unable to Update Malware DB')
+        except Exception:
+            logger.exception('[ERROR] Malware DB Update')
+
+    def update_maltrail_db(self):
+        """Check for update in maltrail DB."""
+        try:
+            mal_db = self.maltrail
+            resp = update_local_db(
+                'Maltrail',
+                settings.MALTRAIL_DB_URL, mal_db)
+            if not resp:
+                return
+            # DB needs update
+            # Check2: DB Syntax Changed
+            lines = resp.decode('utf-8', 'ignore').splitlines()
+            if len(lines) > 100:
+                logger.info('Updating Maltrail Database')
+                with open(mal_db, 'wb') as wfp:
+                    wfp.write(resp)
+            else:
+                logger.warning('Unable to Update Maltrail DB')
+        except Exception:
+            logger.exception('[ERROR] Maltrail DB Update')
+
+    def gelocation(self):
+        """Perform Geolocation."""
+        try:
+            self.IP2Loc.open(self.iplocbin)
+            for domain in self.domainlist:
+                # Tag Good Domains
+                if domain not in self.result:
+                    tmp_d = {}
+                    tmp_d['bad'] = 'no'
+                    self.result[domain] = tmp_d
+                # GeoIP
+                ip = None
+                try:
+                    ip = gethostbyname(domain)
+                except (gaierror, UnicodeError):
+                    pass
+                if ip:
+                    rec = self.IP2Loc.get_all(ip)
+                    self.result[domain]['geolocation'] = rec.__dict__
+                else:
+                    self.result[domain]['geolocation'] = None
+        except Exception:
+            logger.exception('Failed to Perform Geolocation')
+        finally:
+            if self.IP2Loc:
+                self.IP2Loc.close()
+
+    def malware_check(self):
+        try:
+            mal_db = self.malwaredomainlist
+            with io.open(mal_db,
+                         mode='r',
+                         encoding='utf8',
+                         errors='ignore') as flip:
+                entry_list = flip.readlines()
+            for entry in entry_list:
+                enlist = entry.split('","')
+                if len(enlist) > 5:
+                    details_dict = {}
+                    details_dict['domain_or_url'] = enlist[1]
+                    details_dict['ip'] = enlist[2]
+                    details_dict['desc'] = enlist[4]
+                    details_dict['bad'] = 'yes'
+                    dmn_url = details_dict['domain_or_url']
+                    for domain in self.domainlist:
+                        dmn_neturl = get_netloc(dmn_url)
+                        if (((dmn_neturl == domain or dmn_neturl == domain[4:])
+                                and (len(dmn_url) > 1))
+                                or details_dict['ip'].startswith(domain)):
+                            self.result[domain] = details_dict
+        except Exception:
+            logger.exception('[ERROR] Performing Malware Check')
+
+    def maltrail_check(self):
+        try:
+            mal_db = self.maltrail
+            with io.open(mal_db,
+                         mode='r',
+                         encoding='utf8',
+                         errors='ignore') as flip:
+                entry_list = flip.read().splitlines()
+            for domain in self.domainlist:
+                if domain in entry_list:
+                    self.result[domain] = {
+                        'domain_or_url': domain,
+                        'ip': 'N/A',
+                        'desc': 'Malicious Domain tagged by Maltrail',
+                        'bad': 'yes',
+                    }
+        except Exception:
+            logger.exception('[ERROR] Performing Maltrail Check')
+
+    def update(self):
+        if is_internet_available():
+            self.update_maltrail_db()
+        else:
+            logger.warning('Internet not available. '
+                           'Skipping Malware Database Update.')
+
+    def scan(self, urls):
+        if not settings.DOMAIN_MALWARE_SCAN:
+            logger.info('Domain Malware Check disabled in settings')
+            return self.result
+        self.domainlist = get_domains(urls)
+        if self.domainlist:
+            self.update()
+            self.malware_check()
+            self.maltrail_check()
+            self.gelocation()
+        return self.result
+
+
+# Helper Functions
+
+def verify_domain(checkeddom):
+    try:
+        if (len(checkeddom) > 2
+                and '.' in checkeddom
+                and (checkeddom.endswith('.') is False
+                     and re.search('[a-zA-Z0-9]', checkeddom))):
+            return True
+        else:
+            return False
+    except Exception:
+        logger.exception('[ERROR] Verifying Domain')
+
+
+def get_netloc(url):
+    """Get Domain."""
+    try:
+        domain = ''
+        parse_uri = urlparse(url)
+        if not parse_uri.scheme:
+            url = '//' + url
+            parse_uri = urlparse(url)
+        domain = '{uri.netloc}'.format(uri=parse_uri)
+        if verify_domain(domain):
+            return domain
+    except Exception:
+        logger.exception('[ERROR] Extracting Domain form URL')
+
+
+def sanitize_domain(domain):
+    """Sanitize domain to be RFC1034 compliant."""
+    domain = domain.split('_')[0]
+    domain = re.sub(r'[^\w^\.^\-]', '', domain)
+    if domain.startswith('-'):
+        domain = sanitize_domain(domain[1:])
+    elif domain.endswith('-'):
+        domain = sanitize_domain(domain[:-1])
+    return domain
+
+
+def get_domains(urls):
+    """Get Domains."""
+    try:
+        domains = set()
+        for url in urls:
+            parse_uri = urlparse(url)
+            if not parse_uri.scheme:
+                url = '//' + url
+                parse_uri = urlparse(url)
+            domain = sanitize_domain(
+                '{uri.hostname}'.format(uri=parse_uri))
+            if verify_domain(domain):
+                domains.add(domain)
+        return domains
+    except Exception:
+        logger.exception('[ERROR] Extracting Domain form URL')

package/MobSF/mobsf/MalwareAnalyzer/views/Trackers.py

@@ -0,0 +1,237 @@
+import glob
+import io
+import itertools
+import json
+import logging
+import os
+import re
+import subprocess
+from collections import namedtuple
+
+from django.conf import settings
+
+from mobsf.MobSF.utils import (
+    find_java_binary,
+    is_file_exists,
+    is_internet_available,
+    update_local_db,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class Trackers:
+    def __init__(self, apk_dir, tools_dir):
+        self.apk = None
+        self.apk_dir = apk_dir
+        self.tracker_db = os.path.join(
+            settings.SIGNATURE_DIR,
+            'exodus_trackers')
+        self.signatures = None
+        self.nb_trackers_signature = 0
+        self.compiled_tracker_signature = None
+        self.compiled_network_tracker_sig = None
+        self.classes = None
+        self.tools_dir = tools_dir
+        self._update_tracker_db()
+
+    def _update_tracker_db(self):
+        """Update Trackers DB."""
+        try:
+            if not is_internet_available():
+                logger.warning('No Internet Connection. '
+                               'Skipping Trackers Database Update.')
+                return
+            exodus_db = '{}/api/trackers'.format(settings.EXODUS_URL)
+            resp = update_local_db('Trackers',
+                                   exodus_db,
+                                   self.tracker_db)
+            # Check1: SHA256 Change
+            if resp:
+                # DB needs update
+                # Check2: DB Syntax Changed
+                data = json.loads(resp.decode('utf-8', 'ignore'))
+                is_db_format_good = False
+                if 'trackers' in data:
+                    if '1' in data['trackers']:
+                        if 'code_signature' in data['trackers']['1']:
+                            is_db_format_good = True
+                if is_db_format_good:
+                    # DB Format is not changed. Let's update DB
+                    logger.info('Updating Trackers Database....')
+                    with open(self.tracker_db, 'wb') as wfp:
+                        wfp.write(resp)
+                else:
+                    logger.info('Trackers Database format from '
+                                'reports.exodus-privacy.eu.org has changed.'
+                                ' Database is not updated. '
+                                'Please report to: https://github.com/MobSF/'
+                                'Mobile-Security-Framework-MobSF/issues')
+        except Exception:
+            logger.exception('[ERROR] Trackers DB Update')
+
+    def _compile_signatures(self):
+        """
+        Compile Signatures.
+
+        Compiles the regex associated to each signature, in order to speed up
+        the trackers detection.
+        :return: A compiled list of signatures.
+        """
+        self.compiled_tracker_signature = []
+        self.compiled_network_tracker_sig = []
+        try:
+            self.compiled_tracker_signature = [
+                re.compile(track.code_signature)
+                for track in self.signatures]
+            self.compiled_network_tracker_sig = [
+                re.compile(track.network_signature)
+                for track in self.signatures]
+        except TypeError:
+            logger.exception('compiling tracker signature failed')
+
+    def load_trackers_signatures(self):
+        """
+        Load trackers signatures from the official Exodus database.
+
+        :return: a dictionary of signatures.
+        """
+        self.signatures = []
+        with io.open(self.tracker_db,
+                     mode='r',
+                     encoding='utf8',
+                     errors='ignore') as flip:
+            data = json.loads(flip.read())
+        for elm in data['trackers']:
+            self.signatures.append(
+                namedtuple('tracker',
+                           data['trackers'][elm].keys())(
+                               *data['trackers'][elm].values()))
+        self._compile_signatures()
+        self.nb_trackers_signature = len(self.signatures)
+
+    def get_embedded_classes(self):
+        """
+        Get the list of Java classes from all DEX files.
+
+        :return: list of Java classes
+        """
+        if self.classes is not None:
+            return self.classes
+        for dex_file in glob.iglob(os.path.join(self.apk_dir, '*.dex')):
+            if (len(settings.BACKSMALI_BINARY) > 0
+                    and is_file_exists(settings.BACKSMALI_BINARY)):
+                bs_path = settings.BACKSMALI_BINARY
+            else:
+                bs_path = os.path.join(self.tools_dir, 'baksmali-2.5.2.jar')
+            args = [find_java_binary(), '-jar',
+                    bs_path, 'list', 'classes', dex_file]
+            classes = subprocess.check_output(
+                args, universal_newlines=True).splitlines()
+            if self.classes is not None:
+                self.classes = self.classes + classes
+            else:
+                self.classes = classes
+        return self.classes
+
+    def detect_trackers_in_list(self, class_list, network=False):
+        """
+        Detect embedded trackers in the provided classes list/urls.
+
+        :return: list of embedded trackers
+        """
+        if self.signatures is None:
+            self.load_trackers_signatures()
+
+        def _detect_tracker(sig, tracker, class_list):
+            for clazz in class_list:
+                if sig.search(clazz):
+                    return tracker
+            return None
+
+        results = []
+        if network:
+            compiled = self.compiled_network_tracker_sig
+            args = [
+                (compiled[index], tracker, class_list)
+                for (index, tracker) in enumerate(self.signatures) if
+                len(tracker.network_signature) > 3]
+
+        else:
+            compiled = self.compiled_tracker_signature
+            args = [
+                (compiled[index], tracker, class_list)
+                for (index, tracker) in enumerate(self.signatures) if
+                len(tracker.code_signature) > 3]
+
+        for res in itertools.starmap(_detect_tracker, args):
+            if res:
+                results.append(res)
+
+        trackers = [t for t in results if t is not None]
+        trackers = sorted(trackers, key=lambda trackers: trackers.name)
+        return trackers
+
+    def detect_trackers(self):
+        """
+        Detect embedded trackers.
+
+        :return: list of embedded trackers
+        """
+        if self.signatures is None:
+            self.load_trackers_signatures()
+        eclasses = self.get_embedded_classes()
+        if eclasses:
+            return self.detect_trackers_in_list(eclasses)
+        return []
+
+    def detect_runtime_trackers(self, items, deps=False):
+        """
+        Detect runtime trackers.
+
+        :return: list of embedded trackers
+        """
+        if self.signatures is None:
+            self.load_trackers_signatures()
+        if items and not deps:
+            # Domains
+            return self.detect_trackers_in_list(items.keys(), True)
+        elif items and deps:
+            # Runtime Dependencies
+            return self.detect_trackers_in_list(items)
+        return []
+
+    def get_trackers(self):
+        """Get Trackers."""
+        logger.info('Detecting Trackers')
+        trackers = self.detect_trackers()
+        tracker_dict = {'detected_trackers': len(trackers),
+                        'total_trackers': self.nb_trackers_signature,
+                        'trackers': []}
+        for trk in trackers:
+            trk_url = '{}/trackers/{}'.format(settings.EXODUS_URL, trk.id)
+            tracker_dict['trackers'].append({
+                'name': trk.name,
+                'categories': ', '.join(trk.categories),
+                'url': trk_url,
+            })
+        return tracker_dict
+
+    def get_runtime_trackers(self, domains, deps):
+        """Get Trackers from Runtime."""
+        trackers = []
+        logger.info('Detecting Trackers from HTTP(s) traffic')
+        network = self.detect_runtime_trackers(domains)
+        logger.info('Detecting Trackers from Runtime dependencies')
+        runtime = self.detect_runtime_trackers(deps, True)
+        for i in runtime:
+            if i not in network:
+                network.append(i)
+        for trk in network:
+            trk_url = '{}/trackers/{}'.format(settings.EXODUS_URL, trk.id)
+            trackers.append({
+                'name': trk.name,
+                'categories': ', '.join(trk.categories),
+                'url': trk_url,
+            })
+        return trackers

package/MobSF/mobsf/MalwareAnalyzer/views/VirusTotal.py

@@ -0,0 +1,140 @@
+import logging
+
+import requests
+
+from django.conf import settings
+
+from mobsf.MobSF.utils import (
+    file_size,
+    get_config_loc,
+    upstream_proxy,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class VirusTotal:
+
+    base_url = settings.VIRUS_TOTAL_BASE_URL
+
+    def get_report(self, file_hash):
+        """
+        Get Report from VT.
+
+        :param file_hash: md5/sha1/sha256
+        :return: json response / None
+        """
+        try:
+            url = self.base_url + 'report'
+            params = {
+                'apikey': settings.VT_API_KEY,
+                'resource': file_hash}
+            headers = {'Accept-Encoding': 'gzip, deflate'}
+            try:
+                proxies, verify = upstream_proxy('https')
+            except Exception:
+                logger.exception('Setting upstream proxy')
+            try:
+                response = requests.get(
+                    url,
+                    params=params,
+                    headers=headers,
+                    proxies=proxies,
+                    verify=verify)
+                if response.status_code == 403:
+                    logger.error(
+                        'VirusTotal Permission denied, wrong api key?')
+                    return None
+            except Exception:
+                logger.error(
+                    'VirusTotal ConnectionError, check internet connectivity')
+                return None
+            try:
+                json_response = response.json()
+                return json_response
+            except ValueError:
+                return None
+        except Exception:
+            logger.exception('VirusTotal get_report')
+            return None
+
+    def upload_file(self, file_path):
+        """
+        Upload File to VT.
+
+        :param file_path: file path to upload
+        :return: json response / None
+        """
+        try:
+            url = self.base_url + 'scan'
+            if file_size(file_path) > 31:
+                logger.warning('VirusTotal Public API does '
+                               'not support files above 32 MB')
+                return None
+            files = {'file': open(file_path, 'rb')}
+            headers = {'apikey': settings.VT_API_KEY}
+            try:
+                proxies, verify = upstream_proxy('https')
+            except Exception:
+                logger.exception('Setting upstream proxy')
+            try:
+                response = requests.post(
+                    url,
+                    files=files,
+                    data=headers,
+                    proxies=proxies,
+                    verify=verify)
+                if response.status_code == 403:
+                    logger.error(
+                        'VirusTotal Permission denied, wrong api key?')
+                    return None
+            except Exception:
+                logger.error(
+                    'VirusTotal Connection Error, check internet connectivity')
+                return None
+            json_response = response.json()
+            return json_response
+
+        except Exception:
+            logger.exception('VirusTotal upload_file')
+            return None
+
+    def get_result(self, file_path, file_hash):
+        """
+        Get Results from VT.
+
+        Uploading a file and getting the approval msg from VT
+        or fetching existing report
+        :param file_path: file's path
+        :param file_hash: file's hash - md5/sha1/sha256
+        :return: VirusTotal result json / None upon error
+        """
+        try:
+            logger.info('VirusTotal: Check for existing report')
+            report = self.get_report(file_hash)
+            # Check for existing report
+            if report:
+                if report['response_code'] == 1:
+                    logger.info('VirusTotal: %s', report['verbose_msg'])
+                    return report
+            if settings.VT_UPLOAD:
+                logger.info('VirusTotal: file upload')
+                upload_response = self.upload_file(file_path)
+                if upload_response:
+                    logger.info('VirusTotal: %s',
+                                upload_response['verbose_msg'])
+                return upload_response
+            else:
+                logger.info('VirusTotal Scan not performed as file'
+                            ' upload is disabled in %s. '
+                            'To enable file upload, '
+                            'set VT_UPLOAD to True.', get_config_loc())
+                message = ('Scan not performed, VirusTotal file'
+                           f' upload disabled in {get_config_loc()}')
+                report = {
+                    'verbose_msg': message,
+                    'positives': 0,
+                    'total': 0}
+                return report
+        except Exception:
+            logger.exception('VirusTotal get_result')

package/MobSF/mobsf/MalwareAnalyzer/views/apkid.py

@@ -0,0 +1,61 @@
+# -*- coding: utf_8 -*-
+import logging
+import os
+
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+def apkid_analysis(app_dir, apk_file, apk_name):
+    """APKID Analysis of DEX files."""
+    if not settings.APKID_ENABLED:
+        return {}
+    try:
+        import apkid
+    except ImportError:
+        logger.error('APKiD - Could not import APKiD')
+        return {}
+    if not os.path.exists(apk_file):
+        logger.error('APKiD - APK not found')
+        return {}
+
+    apkid_ver = apkid.__version__
+    from apkid.apkid import Scanner, Options
+    from apkid.output import OutputFormatter
+    from apkid.rules import RulesManager
+
+    logger.info('Running APKiD %s', apkid_ver)
+    options = Options(
+        timeout=30,
+        verbose=False,
+        entry_max_scan_size=settings.DATA_UPLOAD_MAX_MEMORY_SIZE,
+        recursive=True,
+    )
+    output = OutputFormatter(
+        json_output=True,
+        output_dir=None,
+        rules_manager=RulesManager(),
+        include_types=False,
+    )
+    rules = options.rules_manager.load()
+    scanner = Scanner(rules, options)
+    res = scanner.scan_file(apk_file)
+    try:
+        findings = output._build_json_output(res)['files']
+    except AttributeError:
+        # apkid >= 2.0.3
+        try:
+            findings = output.build_json_output(res)['files']
+        except AttributeError:
+            logger.error('yara-python dependency required by '
+                         'APKiD is not installed properly. '
+                         'Skipping APKiD analysis!')
+            findings = {}
+    sanitized = {}
+    for item in findings:
+        filename = item['filename']
+        if '!' in filename:
+            filename = filename.split('!', 1)[1]
+        sanitized[filename] = item['matches']
+    return sanitized