supplier-meesho.webflow 1.0.0

package/MobSF/mobsf/StaticAnalyzer/views/shared_func.py

@@ -0,0 +1,424 @@
+# -*- coding: utf_8 -*-
+"""
+Shared Functions.
+
+Module providing the shared functions for static analysis of iOS and Android
+"""
+import hashlib
+import io
+import json
+import logging
+import os
+import platform
+import re
+import shutil
+import subprocess
+import zipfile
+from urllib.parse import urlparse
+from pathlib import Path
+
+import requests
+
+from django.http import HttpResponse
+from django.template.loader import get_template
+from django.utils import timezone
+from django.utils.html import escape
+
+import mobsf.MalwareAnalyzer.views.VirusTotal as VirusTotal
+from mobsf.MobSF import settings
+from mobsf.MobSF.utils import (
+    is_md5,
+    print_n_send_error_response,
+    upstream_proxy,
+)
+from mobsf.StaticAnalyzer.models import (
+    RecentScansDB,
+    StaticAnalyzerAndroid,
+    StaticAnalyzerIOS,
+    StaticAnalyzerWindows,
+)
+from mobsf.StaticAnalyzer.views.comparer import generic_compare
+from mobsf.StaticAnalyzer.views.android.db_interaction import (
+    get_context_from_db_entry as adb)
+from mobsf.StaticAnalyzer.views.ios.db_interaction import (
+    get_context_from_db_entry as idb)
+from mobsf.StaticAnalyzer.views.windows.db_interaction import (
+    get_context_from_db_entry as wdb)
+
+logger = logging.getLogger(__name__)
+try:
+    import pdfkit
+except ImportError:
+    logger.warning(
+        'wkhtmltopdf is not installed/configured properly.'
+        ' PDF Report Generation is disabled')
+logger = logging.getLogger(__name__)
+ctype = 'application/json; charset=utf-8'
+
+
+def hash_gen(app_path) -> tuple:
+    """Generate and return sha1 and sha256 as a tuple."""
+    try:
+        logger.info('Generating Hashes')
+        sha1 = hashlib.sha1()
+        sha256 = hashlib.sha256()
+        block_size = 65536
+        with io.open(app_path, mode='rb') as afile:
+            buf = afile.read(block_size)
+            while buf:
+                sha1.update(buf)
+                sha256.update(buf)
+                buf = afile.read(block_size)
+        sha1val = sha1.hexdigest()
+        sha256val = sha256.hexdigest()
+        return sha1val, sha256val
+    except Exception:
+        logger.exception('Generating Hashes')
+
+
+def unzip(app_path, ext_path):
+    logger.info('Unzipping')
+    try:
+        files = []
+        with zipfile.ZipFile(app_path, 'r') as zipptr:
+            for fileinfo in zipptr.infolist():
+                filename = fileinfo.filename
+                if not isinstance(filename, str):
+                    filename = str(
+                        filename, encoding='utf-8', errors='replace')
+                files.append(filename)
+                zipptr.extract(filename, ext_path)
+        return files
+    except Exception:
+        logger.exception('Unzipping Error')
+        if platform.system() == 'Windows':
+            logger.info('Not yet Implemented.')
+        else:
+            logger.info('Using the Default OS Unzip Utility.')
+            try:
+                unzip_b = shutil.which('unzip')
+                subprocess.call(
+                    [unzip_b, '-o', '-q', app_path, '-d', ext_path])
+                dat = subprocess.check_output([unzip_b, '-qq', '-l', app_path])
+                dat = dat.decode('utf-8').split('\n')
+                files_det = ['Length   Date   Time   Name']
+                files_det = files_det + dat
+                return files_det
+            except Exception:
+                logger.exception('Unzipping Error')
+
+
+def pdf(request, api=False, jsonres=False):
+    try:
+        if api:
+            checksum = request.POST['hash']
+        else:
+            checksum = request.GET['md5']
+        hash_match = re.match('^[0-9a-f]{32}$', checksum)
+        if not hash_match:
+            if api:
+                return {'error': 'Invalid scan hash'}
+            else:
+                return HttpResponse(
+                    json.dumps({'md5': 'Invalid scan hash'}),
+                    content_type=ctype, status=500)
+        # Do Lookups
+        android_static_db = StaticAnalyzerAndroid.objects.filter(
+            MD5=checksum)
+        ios_static_db = StaticAnalyzerIOS.objects.filter(
+            MD5=checksum)
+        win_static_db = StaticAnalyzerWindows.objects.filter(
+            MD5=checksum)
+
+        if android_static_db.exists():
+            context, template = handle_pdf_android(android_static_db)
+        elif ios_static_db.exists():
+            context, template = handle_pdf_ios(ios_static_db)
+        elif win_static_db.exists():
+            context, template = handle_pdf_win(win_static_db)
+        else:
+            if api:
+                return {'report': 'Report not Found'}
+            else:
+                return HttpResponse(
+                    json.dumps({'report': 'Report not Found'}),
+                    content_type=ctype,
+                    status=500)
+        # Do VT Scan only on binaries
+        context['virus_total'] = None
+        ext = os.path.splitext(context['file_name'].lower())[1]
+        if settings.VT_ENABLED and ext != '.zip':
+            app_bin = os.path.join(
+                settings.UPLD_DIR,
+                checksum + '/',
+                checksum + ext)
+            vt = VirusTotal.VirusTotal()
+            context['virus_total'] = vt.get_result(app_bin, checksum)
+        # Get Local Base URL
+        proto = 'file://'
+        host_os = 'nix'
+        if platform.system() == 'Windows':
+            proto = 'file:///'
+            host_os = 'windows'
+        context['base_url'] = proto + settings.BASE_DIR
+        context['dwd_dir'] = proto + settings.DWD_DIR
+        context['host_os'] = host_os
+        context['timestamp'] = RecentScansDB.objects.get(
+            MD5=checksum).TIMESTAMP
+        try:
+            if api and jsonres:
+                return {'report_dat': context}
+            else:
+                options = {
+                    'page-size': 'Letter',
+                    'quiet': '',
+                    'enable-local-file-access': '',
+                    'no-collate': '',
+                    'margin-top': '0.50in',
+                    'margin-right': '0.50in',
+                    'margin-bottom': '0.50in',
+                    'margin-left': '0.50in',
+                    'encoding': 'UTF-8',
+                    'custom-header': [
+                        ('Accept-Encoding', 'gzip'),
+                    ],
+                    'no-outline': None,
+                }
+                # Added proxy support to wkhtmltopdf
+                proxies, _ = upstream_proxy('https')
+                if proxies['https']:
+                    options['proxy'] = proxies['https']
+                html = template.render(context)
+                pdf_dat = pdfkit.from_string(html, False, options=options)
+                if api:
+                    return {'pdf_dat': pdf_dat}
+                return HttpResponse(pdf_dat,
+                                    content_type='application/pdf')
+        except Exception as exp:
+            logger.exception('Error Generating PDF Report')
+            if api:
+                return {
+                    'error': 'Cannot Generate PDF/JSON',
+                    'err_details': str(exp)}
+            else:
+                err = {
+                    'pdf_error': 'Cannot Generate PDF',
+                    'err_details': str(exp)}
+                return HttpResponse(
+                    json.dumps(err),  # lgtm [py/stack-trace-exposure]
+                    content_type=ctype,
+                    status=500)
+    except Exception as exp:
+        logger.exception('Error Generating PDF Report')
+        msg = str(exp)
+        exp = exp.__doc__
+        if api:
+            return print_n_send_error_response(request, msg, True, exp)
+        else:
+            return print_n_send_error_response(request, msg, False, exp)
+
+
+def handle_pdf_android(static_db):
+    logger.info(
+        'Fetching data from DB for '
+        'PDF Report Generation (Android)')
+    context = adb(static_db)
+    context['average_cvss'], context[
+        'security_score'] = score(context['code_analysis'])
+    if context['file_name'].lower().endswith('.zip'):
+        logger.info('Generating PDF report for android zip')
+        template = get_template(
+            'pdf/android_report.html')
+    else:
+        logger.info('Generating PDF report for android apk')
+        template = get_template(
+            'pdf/android_report.html')
+    return context, template
+
+
+def handle_pdf_ios(static_db):
+    logger.info('Fetching data from DB for '
+                'PDF Report Generation (IOS)')
+    context = idb(static_db)
+    if context['file_name'].lower().endswith('.zip'):
+        logger.info('Generating PDF report for IOS zip')
+        context['average_cvss'], context[
+            'security_score'] = score(context['code_analysis'])
+        template = get_template(
+            'pdf/ios_report.html')
+    else:
+        logger.info('Generating PDF report for IOS ipa')
+        context['average_cvss'], context[
+            'security_score'] = score(
+                context['binary_analysis'])
+        template = get_template(
+            'pdf/ios_report.html')
+    return context, template
+
+
+def handle_pdf_win(static_db):
+    logger.info(
+        'Fetching data from DB for '
+        'PDF Report Generation (APPX)')
+    context = wdb(static_db)
+    template = get_template(
+        'pdf/windows_report.html')
+    return context, template
+
+
+def url_n_email_extract(dat, relative_path):
+    """Extract URLs and Emails from Source Code."""
+    urls = []
+    emails = []
+    urllist = []
+    url_n_file = []
+    email_n_file = []
+    # URLs Extraction My Custom regex
+    pattern = re.compile(
+        (
+            r'((?:https?://|s?ftps?://|'
+            r'file://|javascript:|data:|www\d{0,3}[.])'
+            r'[\w().=/;,#:@?&~*+!$%\'{}-]+)'
+        ),
+        re.UNICODE)
+    urllist = re.findall(pattern, dat)
+    uflag = 0
+    for url in urllist:
+        if url not in urls:
+            urls.append(url)
+            uflag = 1
+    if uflag == 1:
+        url_n_file.append(
+            {'urls': urls, 'path': escape(relative_path)})
+
+    # Email Extraction Regex
+    regex = re.compile(r'[\w.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}')
+    eflag = 0
+    for email in regex.findall(dat.lower()):
+        if (email not in emails) and (not email.startswith('//')):
+            emails.append(email)
+            eflag = 1
+    if eflag == 1:
+        email_n_file.append(
+            {'emails': emails, 'path': escape(relative_path)})
+    return urllist, url_n_file, email_n_file
+
+
+# This is just the first sanity check that triggers generic_compare
+def compare_apps(request, hash1: str, hash2: str, api=False):
+    if hash1 == hash2:
+        error_msg = 'Results with same hash cannot be compared'
+        return print_n_send_error_response(request, error_msg, api)
+    # Second Validation for REST API
+    if not (is_md5(hash1) and is_md5(hash2)):
+        error_msg = 'Invalid hashes'
+        return print_n_send_error_response(request, error_msg, api)
+    logger.info(
+        'Starting App compare for %s and %s', hash1, hash2)
+    return generic_compare(request, hash1, hash2, api)
+
+
+def score(findings):
+    # Score Apps based on AVG CVSS Score
+    cvss_scores = []
+    avg_cvss = 0
+    app_score = 100
+    for finding in findings.values():
+        find = finding.get('metadata')
+        if not find:
+            # Hack to support iOS Binary Scan Results
+            find = finding
+        if find.get('cvss'):
+            if find['cvss'] != 0:
+                cvss_scores.append(find['cvss'])
+        if find['severity'] == 'high':
+            app_score = app_score - 15
+        elif find['severity'] == 'warning':
+            app_score = app_score - 10
+        elif find['severity'] == 'good':
+            app_score = app_score + 5
+    if cvss_scores:
+        avg_cvss = round(sum(cvss_scores) / len(cvss_scores), 1)
+    if app_score < 0:
+        app_score = 10
+    elif app_score > 100:
+        app_score = 100
+    return avg_cvss, app_score
+
+
+def update_scan_timestamp(scan_hash):
+    # Update the last scan time.
+    tms = timezone.now()
+    RecentScansDB.objects.filter(MD5=scan_hash).update(TIMESTAMP=tms)
+
+
+def open_firebase(url):
+    # Detect Open Firebase Database
+    try:
+        purl = urlparse(url)
+        base_url = '{}://{}/.json'.format(purl.scheme, purl.netloc)
+        proxies, verify = upstream_proxy('https')
+        headers = {
+            'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1)'
+                           ' AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/39.0.2171.95 Safari/537.36')}
+        resp = requests.get(base_url, headers=headers,
+                            proxies=proxies, verify=verify)
+        if resp.status_code == 200:
+            return base_url, True
+    except Exception:
+        logger.warning('Open Firebase DB detection failed.')
+    return url, False
+
+
+def firebase_analysis(urls):
+    # Detect Firebase URL
+    firebase_db = []
+    logger.info('Detecting Firebase URL(s)')
+    for url in urls:
+        if 'firebaseio.com' in url:
+            returl, is_open = open_firebase(url)
+            fbdic = {'url': returl, 'open': is_open}
+            if fbdic not in firebase_db:
+                firebase_db.append(fbdic)
+    return firebase_db
+
+
+def find_java_source_folder(base_folder: Path):
+    # Find the correct java/kotlin source folder for APK/source zip
+    # Returns a Tuple of - (SRC_PATH, SRC_TYPE, SRC_SYNTAX)
+    return next(p for p in [(base_folder / 'java_source',
+                             'java', '*.java'),
+                            (base_folder / 'app' / 'src' / 'main' / 'java',
+                             'java', '*.java'),
+                            (base_folder / 'app' / 'src' / 'main' / 'kotlin',
+                             'kotlin', '*.kt'),
+                            (base_folder / 'src',
+                             'java', '*.java')]
+                if p[0].exists())
+
+
+def is_secret(inp):
+    inp = inp.lower()
+    """Check if captures string is a possible secret."""
+    iden = (
+        'api"', 'key"', 'api_', 'key_', 'secret"',
+        'password"', 'aws', 'gcp', 's3_', '_s3', 'secret_',
+        'token"', 'username"', 'user_name"', 'user"',
+        'bearer', 'jwt', 'certificate"', 'credential',
+        'azure', 'webhook', 'twilio_', 'bitcoin',
+        '_auth', 'firebase', 'oauth', 'authorization',
+        'private', 'pwd', 'session', 'token_',
+    )
+    not_string = (
+        'label_', 'text', 'hint', 'msg_', 'create_',
+        'message', 'new', 'confirm', 'activity_',
+        'forgot', 'dashboard_', 'current_', 'signup',
+        'sign_in', 'signin', 'title_', 'welcome_',
+        'change_', 'this_', 'the_', 'placeholder',
+        'invalid_', 'btn_', 'action_', 'prompt_',
+        'lable', 'hide_', 'old', 'update', 'error',
+        'empty', 'txt_', 'lbl_',
+    )
+    not_str = any(i in inp for i in not_string)
+    return any(i in inp for i in iden) and not not_str

package/MobSF/mobsf/StaticAnalyzer/views/windows/db_interaction.py

@@ -0,0 +1,130 @@
+"""Module holding the functions for the db."""
+import logging
+
+from django.conf import settings
+
+from mobsf.MobSF.utils import python_list
+from mobsf.StaticAnalyzer.models import StaticAnalyzerWindows
+from mobsf.StaticAnalyzer.models import RecentScansDB
+
+logger = logging.getLogger(__name__)
+
+
+def get_context_from_db_entry(db_entry):
+    """Return the context for APPX from DB."""
+    try:
+        logger.info('Analysis is already Done. Fetching data from the DB...')
+        context = {
+            'title': 'Static Analysis',
+            'version': settings.MOBSF_VER,
+            'file_name': db_entry[0].FILE_NAME,
+            'app_name': db_entry[0].APP_NAME,
+            'publisher_name': db_entry[0].PUBLISHER_NAME,
+            'size': db_entry[0].SIZE,
+            'md5': db_entry[0].MD5,
+            'sha1': db_entry[0].SHA1,
+            'sha256': db_entry[0].SHA256,
+            'app_version': db_entry[0].APP_VERSION,
+            'architecture': db_entry[0].ARCHITECTURE,
+            'compiler_version': db_entry[0].COMPILER_VERSION,
+            'visual_studio_version': db_entry[0].VISUAL_STUDIO_VERSION,
+            'visual_studio_edition': db_entry[0].VISUAL_STUDIO_EDITION,
+            'target_os': db_entry[0].TARGET_OS,
+            'appx_dll_version': db_entry[0].APPX_DLL_VERSION,
+            'proj_guid': db_entry[0].PROJ_GUID,
+            'opti_tool': db_entry[0].OPTI_TOOL,
+            'target_run': db_entry[0].TARGET_RUN,
+            'files': python_list(db_entry[0].FILES),
+            'strings': python_list(db_entry[0].STRINGS),
+            'binary_analysis': python_list(db_entry[0].BINARY_ANALYSIS),
+            'binary_warnings': python_list(db_entry[0].BINARY_WARNINGS),
+        }
+        return context
+    except Exception:
+        logger.exception('Fetching from DB')
+
+
+def get_context_from_analysis(app_dic,
+                              xml_dic,
+                              bin_an_dic):
+    """Get the context for APPX from analysis results."""
+    try:
+        context = {
+            'title': 'Static Analysis',
+            'version': settings.MOBSF_VER,
+            'file_name': app_dic['app_name'],
+            'app_name': bin_an_dic['bin_name'],
+            'publisher_name': xml_dic['pub_name'],
+            'size': app_dic['size'],
+            'md5': app_dic['md5'],
+            'sha1': app_dic['sha1'],
+            'sha256': app_dic['sha256'],
+            'app_version': xml_dic['version'],
+            'architecture': xml_dic['arch'],
+            'compiler_version': xml_dic['compiler_version'],
+            'visual_studio_version': xml_dic['visual_studio_version'],
+            'visual_studio_edition': xml_dic['visual_studio_edition'],
+            'target_os': xml_dic['target_os'],
+            'appx_dll_version': xml_dic['appx_dll_version'],
+            'proj_guid': xml_dic['proj_guid'],
+            'opti_tool': xml_dic['opti_tool'],
+            'target_run': xml_dic['target_run'],
+            'files': app_dic['files'],
+            'strings': bin_an_dic['strings'],
+            'binary_analysis': bin_an_dic['results'],
+            'binary_warnings': bin_an_dic['warnings'],
+        }
+        return context
+    except Exception:
+        logger.exception('Rendering to Template')
+
+
+def save_or_update(update_type,
+                   app_dic,
+                   xml_dic,
+                   bin_an_dic) -> None:
+    """Save/Update an APPX DB entry."""
+    try:
+        values = {
+            'FILE_NAME': app_dic['app_name'],
+            'APP_NAME': bin_an_dic['bin_name'],
+            'PUBLISHER_NAME': xml_dic['pub_name'],
+            'SIZE': app_dic['size'],
+            'MD5': app_dic['md5'],
+            'SHA1': app_dic['sha1'],
+            'SHA256': app_dic['sha256'],
+            'APP_VERSION': xml_dic['version'],
+            'ARCHITECTURE': xml_dic['arch'],
+            'COMPILER_VERSION': xml_dic['compiler_version'],
+            'VISUAL_STUDIO_VERSION': xml_dic['visual_studio_version'],
+            'VISUAL_STUDIO_EDITION': xml_dic['visual_studio_edition'],
+            'TARGET_OS': xml_dic['target_os'],
+            'APPX_DLL_VERSION': xml_dic['appx_dll_version'],
+            'PROJ_GUID': xml_dic['proj_guid'],
+            'OPTI_TOOL': xml_dic['opti_tool'],
+            'TARGET_RUN': xml_dic['target_run'],
+            'FILES': app_dic['files'],
+            'STRINGS': bin_an_dic['strings'],
+            'BINARY_ANALYSIS': bin_an_dic['results'],
+            'BINARY_WARNINGS': bin_an_dic['warnings'],
+        }
+        if update_type == 'save':
+            db_entry = StaticAnalyzerWindows.objects.filter(
+                MD5=app_dic['md5'])
+            if not db_entry.exists():
+                StaticAnalyzerWindows.objects.create(**values)
+        else:
+            StaticAnalyzerWindows.objects.filter(
+                MD5=app_dic['md5']).update(**values)
+    except Exception:
+        logger.exception('Updating DB')
+    try:
+        values = {
+            'APP_NAME': bin_an_dic['bin_name'],
+            'PACKAGE_NAME': xml_dic['pub_name'],
+            'VERSION_NAME': xml_dic['version'],
+        }
+        RecentScansDB.objects.filter(
+            MD5=app_dic['md5']).update(**values)
+    except Exception:
+        logger.exception('Updating RecentScansDB')