supplier-meesho.webflow 1.0.0

package/MobSF/mobsf/DynamicAnalyzer/views/android/analysis.py

@@ -0,0 +1,285 @@
+"""Perform Analysis on Dynamic Analysis Data."""
+import io
+import logging
+import os
+import re
+import shutil
+import tarfile
+from json import load
+from pathlib import Path
+
+from mobsf.MobSF.utils import (
+    clean_filename,
+    is_file_exists,
+    is_pipe_or_link,
+    python_list,
+)
+from mobsf.StaticAnalyzer.models import StaticAnalyzerAndroid
+from mobsf.MalwareAnalyzer.views.MalwareDomainCheck import (
+    MalwareDomainCheck,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def run_analysis(apk_dir, md5_hash, package):
+    """Run Dynamic File Analysis."""
+    analysis_result = {}
+    logger.info('Dynamic File Analysis')
+    domains = {}
+    clipboard = []
+    # Collect Log data
+    data = get_log_data(apk_dir, package)
+    clip_tag = 'I/CLIPDUMP-INFO-LOG'
+    clip_tag2 = 'I CLIPDUMP-INFO-LOG'
+    # Collect Clipboard
+    for log_line in data['logcat']:
+        if clip_tag in log_line:
+            clipboard.append(log_line.replace(clip_tag, 'Process ID '))
+        if clip_tag2 in log_line:
+            log_line = log_line.split(clip_tag2)[1]
+            clipboard.append(log_line)
+    # URLs My Custom regex
+    url_pattern = re.compile(
+        r'((?:https?://|s?ftps?://|file://|'
+        r'javascript:|data:|www\d{0,3}'
+        r'[.])[\w().=/;,#:@?&~*+!$%\'{}-]+)', re.UNICODE)
+    urls = re.findall(url_pattern, data['traffic'].lower())
+    if urls:
+        urls = list(set(urls))
+    else:
+        urls = []
+    # Domain Extraction and Malware Check
+    logger.info('Performing Malware Check on extracted Domains')
+    domains = MalwareDomainCheck().scan(urls)
+
+    # Email Etraction Regex
+    emails = []
+    regex = re.compile(r'[\w.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}')
+    for email in regex.findall(data['traffic'].lower()):
+        if (email not in emails) and (not email.startswith('//')):
+            emails.append(email)
+    # Tar dump and fetch files
+    all_files = get_app_files(apk_dir, md5_hash, package)
+    analysis_result['urls'] = urls
+    analysis_result['domains'] = domains
+    analysis_result['emails'] = emails
+    analysis_result['clipboard'] = clipboard
+    analysis_result['xml'] = all_files['xml']
+    analysis_result['sqlite'] = all_files['sqlite']
+    analysis_result['other_files'] = all_files['others']
+    analysis_result['tls_tests'] = get_tls_logs(apk_dir, md5_hash)
+    return analysis_result
+
+
+def get_screenshots(md5_hash, download_dir):
+    """Get Screenshots."""
+    # Only After Download Process is Done
+    result = {}
+    imgs = []
+    act_imgs = []
+    expact_imgs = []
+    act = {}
+    exp_act = {}
+    try:
+        screen_dir = os.path.join(download_dir,
+                                  md5_hash + '-screenshots-apk/')
+        if os.path.exists(screen_dir):
+            for img in os.listdir(screen_dir):
+                if img.endswith('.png'):
+                    if img.startswith('act'):
+                        act_imgs.append(img)
+                    elif img.startswith('expact'):
+                        expact_imgs.append(img)
+                    else:
+                        imgs.append(img)
+            try:
+                sadb = StaticAnalyzerAndroid.objects.get(MD5=md5_hash)
+                exported_act = python_list(sadb.EXPORTED_ACTIVITIES)
+                act_desc = python_list(sadb.ACTIVITIES)
+                if act_imgs:
+                    if len(act_imgs) == len(act_desc):
+                        act = dict(list(zip(act_imgs, act_desc)))
+                if expact_imgs:
+                    if len(expact_imgs) == len(exported_act):
+                        exp_act = dict(list(zip(expact_imgs, exported_act)))
+            except Exception:
+                pass
+                # On device only APK don't have this information available.
+    except Exception:
+        logger.exception('Organising screenshots')
+    result['screenshots'] = imgs
+    result['activities'] = act
+    result['exported_activities'] = exp_act
+    return result
+
+
+def get_tls_logs(apk_dir, md5_hash):
+    """Get TLS/SSL test logs."""
+    out = Path(apk_dir) / 'mobsf_tls_tests.json'
+    if not out.exists():
+        return None
+    with out.open(encoding='utf-8') as src:
+        return load(src)
+
+
+def get_log_data(apk_dir, package):
+    """Get Data for analysis."""
+    logcat_data = []
+    droidmon_data = ''
+    apimon_data = ''
+    frida_logs = ''
+    web_data = ''
+    traffic = ''
+    httptools = os.path.join(str(Path.home()), '.httptools')
+    web = os.path.join(httptools, 'flows', package + '.flow.txt')
+    logcat = os.path.join(apk_dir, 'logcat.txt')
+    xlogcat = os.path.join(apk_dir, 'x_logcat.txt')
+    apimon = os.path.join(apk_dir, 'mobsf_api_monitor.txt')
+    fd_logs = os.path.join(apk_dir, 'mobsf_frida_out.txt')
+    if is_file_exists(web):
+        with io.open(web,
+                     mode='r',
+                     encoding='utf8',
+                     errors='ignore') as flip:
+            web_data = flip.read()
+    if is_file_exists(logcat):
+        with io.open(logcat,  # lgtm [py/path-injection]
+                     mode='r',
+                     encoding='utf8',
+                     errors='ignore') as flip:
+            logcat_data = flip.readlines()
+            traffic = ''.join(logcat_data)
+    if is_file_exists(xlogcat):
+        with io.open(xlogcat,  # lgtm [py/path-injection]
+                     mode='r',
+                     encoding='utf8',
+                     errors='ignore') as flip:
+            droidmon_data = flip.read()
+    if is_file_exists(apimon):
+        with io.open(apimon,  # lgtm [py/path-injection]
+                     mode='r',
+                     encoding='utf8',
+                     errors='ignore') as flip:
+            apimon_data = flip.read()
+    if is_file_exists(fd_logs):
+        with io.open(fd_logs,  # lgtm [py/path-injection]
+                     mode='r',
+                     encoding='utf8',
+                     errors='ignore') as flip:
+            frida_logs = flip.read()
+    traffic = (web_data + traffic + droidmon_data
+               + apimon_data + frida_logs)
+    return {'logcat': logcat_data,
+            'traffic': traffic}
+
+
+def safe_paths(tar_meta):
+    """Safe filenames in windows."""
+    for fh in tar_meta:
+        fh.name = clean_filename(fh.name)
+        yield fh
+
+
+def get_app_files(apk_dir, md5_hash, package):
+    """Get files from device."""
+    logger.info('Getting app files')
+    all_files = {'xml': [], 'sqlite': [], 'others': []}
+    # Extract Device Data
+    tar_loc = os.path.join(apk_dir, package + '.tar')
+    untar_dir = os.path.join(apk_dir, 'DYNAMIC_DeviceData/')
+    if not is_file_exists(tar_loc):
+        return all_files
+    if os.path.exists(untar_dir):
+        # fix for permission errors
+        shutil.rmtree(untar_dir)
+    try:
+        with tarfile.open(tar_loc, errorlevel=1) as tar:
+            tar.extractall(untar_dir, members=safe_paths(tar))
+    except FileExistsError:
+        pass
+    except Exception:
+        logger.exception('Tar extraction failed')
+    # Do Static Analysis on Data from Device
+    try:
+        if not os.path.exists(untar_dir):
+            os.makedirs(untar_dir)
+        for dir_name, _, files in os.walk(untar_dir):
+            for jfile in files:
+                file_path = os.path.join(untar_dir, dir_name, jfile)
+                fileparam = file_path.replace(untar_dir, '')
+                if is_pipe_or_link(file_path):
+                    continue
+                if jfile == 'lib':
+                    pass
+                else:
+                    if jfile.endswith('.xml'):
+                        all_files['xml'].append(
+                            {'type': 'xml', 'file': fileparam})
+                    else:
+                        with open(file_path,  # lgtm [py/path-injection]
+                                  'r',
+                                  encoding='ISO-8859-1') as flip:
+                            file_cnt_sig = flip.read(6)
+                        if file_cnt_sig == 'SQLite':
+                            all_files['sqlite'].append(
+                                {'type': 'db', 'file': fileparam})
+                        elif not jfile.endswith('.DS_Store'):
+                            all_files['others'].append(
+                                {'type': 'others', 'file': fileparam})
+    except Exception:
+        logger.exception('Getting app files')
+    return all_files
+
+
+def generate_download(apk_dir, md5_hash, download_dir, package):
+    """Generating Downloads."""
+    logger.info('Generating Downloads')
+    try:
+        httptools = os.path.join(str(Path.home()), '.httptools')
+        logcat = os.path.join(apk_dir, 'logcat.txt')
+        xlogcat = os.path.join(apk_dir, 'x_logcat.txt')
+        apimon = os.path.join(apk_dir, 'mobsf_api_monitor.txt')
+        fd_logs = os.path.join(apk_dir, 'mobsf_frida_out.txt')
+        dumpsys = os.path.join(apk_dir, 'dump.txt')
+        sshot = os.path.join(apk_dir, 'screenshots-apk/')
+        web = os.path.join(httptools, 'flows', package + '.flow.txt')
+        star = os.path.join(apk_dir, package + '.tar')
+
+        dlogcat = os.path.join(download_dir, md5_hash + '-logcat.txt')
+        dxlogcat = os.path.join(download_dir, md5_hash + '-x_logcat.txt')
+        dapimon = os.path.join(download_dir, md5_hash + '-api_monitor.txt')
+        dfd_logs = os.path.join(download_dir, md5_hash + '-frida_out.txt')
+        ddumpsys = os.path.join(download_dir, md5_hash + '-dump.txt')
+        dsshot = os.path.join(download_dir, md5_hash + '-screenshots-apk/')
+        dweb = os.path.join(download_dir, md5_hash + '-web_traffic.txt')
+        dstar = os.path.join(download_dir, md5_hash + '-app_data.tar')
+
+        # Delete existing data
+        dellist = [dlogcat, dxlogcat, dapimon,
+                   dfd_logs, ddumpsys, dsshot,
+                   dweb, dstar]
+        for item in dellist:
+            if os.path.isdir(item):
+                shutil.rmtree(item)
+            elif os.path.isfile(item):
+                os.remove(item)
+        # Copy new data
+        shutil.copyfile(logcat, dlogcat)
+        shutil.copyfile(dumpsys, ddumpsys)
+        if is_file_exists(xlogcat):
+            shutil.copyfile(xlogcat, dxlogcat)
+        if is_file_exists(apimon):
+            shutil.copyfile(apimon, dapimon)
+        if is_file_exists(fd_logs):
+            shutil.copyfile(fd_logs, dfd_logs)
+        try:
+            shutil.copytree(sshot, dsshot)
+        except Exception:
+            pass
+        if is_file_exists(web):
+            shutil.copyfile(web, dweb)
+        if is_file_exists(star):
+            shutil.copyfile(star, dstar)
+    except Exception:
+        logger.exception('Generating Downloads')

package/MobSF/mobsf/DynamicAnalyzer/views/android/dynamic_analyzer.py

@@ -0,0 +1,316 @@
+# -*- coding: utf_8 -*-
+"""Android Dynamic Analysis."""
+import logging
+import os
+import time
+from pathlib import Path
+from json import dump
+
+from shelljob import proc
+
+
+from django.http import (HttpResponseRedirect,
+                         StreamingHttpResponse)
+from django.conf import settings
+from django.shortcuts import render
+
+from mobsf.DynamicAnalyzer.views.android.environment import Environment
+from mobsf.DynamicAnalyzer.views.android.operations import (
+    get_package_name,
+)
+from mobsf.DynamicAnalyzer.tools.webproxy import (
+    get_http_tools_url,
+    start_httptools_ui,
+    stop_httptools,
+)
+from mobsf.MobSF.utils import (
+    get_config_loc,
+    get_device,
+    get_proxy_ip,
+    is_md5,
+    print_n_send_error_response,
+    strict_package_check,
+)
+from mobsf.MobSF.views.scanning import add_to_recent_scan
+from mobsf.StaticAnalyzer.models import StaticAnalyzerAndroid
+
+logger = logging.getLogger(__name__)
+
+
+def dynamic_analysis(request, api=False):
+    """Android Dynamic Analysis Entry point."""
+    try:
+        scan_apps = []
+        device_packages = {}
+        and_ver = None
+        and_sdk = None
+        apks = StaticAnalyzerAndroid.objects.filter(
+            APP_TYPE='apk')
+        for apk in reversed(apks):
+            temp_dict = {
+                'ICON_FOUND': apk.ICON_FOUND,
+                'MD5': apk.MD5,
+                'APP_NAME': apk.APP_NAME,
+                'VERSION_NAME': apk.VERSION_NAME,
+                'FILE_NAME': apk.FILE_NAME,
+                'PACKAGE_NAME': apk.PACKAGE_NAME,
+            }
+            scan_apps.append(temp_dict)
+        try:
+            identifier = get_device()
+        except Exception:
+            msg = ('Is Android VM running? MobSF cannot'
+                   ' find android instance identifier.'
+                   ' Please run an android instance and refresh'
+                   ' this page. If this error persists,'
+                   ' set ANALYZER_IDENTIFIER in '
+                   f'{get_config_loc()}')
+            return print_n_send_error_response(request, msg, api)
+        try:
+            if identifier:
+                env = Environment(identifier)
+                device_packages = env.get_device_packages()
+                pkg_file = Path(settings.DWD_DIR) / 'packages.json'
+                with pkg_file.open('w', encoding='utf-8') as target:
+                    dump(device_packages, target)
+                and_ver = env.get_android_version()
+                and_sdk = env.get_android_sdk()
+        except Exception:
+            pass
+        context = {'apps': scan_apps,
+                   'identifier': identifier,
+                   'android_version': and_ver,
+                   'android_sdk': and_sdk,
+                   'proxy_ip': get_proxy_ip(identifier),
+                   'proxy_port': settings.PROXY_PORT,
+                   'settings_loc': get_config_loc(),
+                   'device_packages': device_packages,
+                   'title': 'MobSF Dynamic Analysis',
+                   'version': settings.MOBSF_VER}
+        if api:
+            return context
+        template = 'dynamic_analysis/dynamic_analysis.html'
+        return render(request, template, context)
+    except Exception as exp:
+        logger.exception('Dynamic Analysis')
+        return print_n_send_error_response(request,
+                                           exp,
+                                           api)
+
+
+def dynamic_analyzer(request, checksum, api=False):
+    """Android Dynamic Analyzer Environment."""
+    try:
+        identifier = None
+        if api:
+            reinstall = request.POST.get('re_install', '1')
+            install = request.POST.get('install', '1')
+        else:
+            reinstall = request.GET.get('re_install', '1')
+            install = request.GET.get('install', '1')
+        if not is_md5(checksum):
+            # We need this check since checksum is not validated
+            # in REST API
+            return print_n_send_error_response(
+                request,
+                'Invalid Parameters',
+                api)
+        package = get_package_name(checksum)
+        if not package:
+            return print_n_send_error_response(
+                request,
+                'Cannot get package name from checksum',
+                api)
+        logger.info('Creating Dynamic Analysis Environment for %s', package)
+        try:
+            identifier = get_device()
+        except Exception:
+            pass
+        if not identifier:
+            msg = ('Is the android instance running? MobSF cannot'
+                   ' find android instance identifier. '
+                   'Please run an android instance and refresh'
+                   ' this page. If this error persists,'
+                   ' set ANALYZER_IDENTIFIER in '
+                   f'{get_config_loc()}')
+            return print_n_send_error_response(request, msg, api)
+        env = Environment(identifier)
+        if not env.connect_n_mount():
+            msg = 'Cannot Connect to ' + identifier
+            return print_n_send_error_response(request, msg, api)
+        version = env.get_android_version()
+        logger.info('Android Version identified as %s', version)
+        xposed_first_run = False
+        if not env.is_mobsfyied(version):
+            msg = ('This Android instance is not MobSFyed/Outdated.\n'
+                   'MobSFying the android runtime environment')
+            logger.warning(msg)
+            if not env.mobsfy_init():
+                return print_n_send_error_response(
+                    request,
+                    'Failed to MobSFy the instance',
+                    api)
+            if version < 5:
+                xposed_first_run = True
+        if xposed_first_run:
+            msg = ('Have you MobSFyed the instance before'
+                   ' attempting Dynamic Analysis?'
+                   ' Install Framework for Xposed.'
+                   ' Restart the device and enable'
+                   ' all Xposed modules. And finally'
+                   ' restart the device once again.')
+            return print_n_send_error_response(request, msg, api)
+        # Clean up previous analysis
+        env.dz_cleanup(checksum)
+        # Configure Web Proxy
+        env.configure_proxy(package, request)
+        # Supported in Android 5+
+        env.enable_adb_reverse_tcp(version)
+        # Apply Global Proxy to device
+        env.set_global_proxy(version)
+        # Start Clipboard monitor
+        env.start_clipmon()
+        # Get Screen Resolution
+        screen_width, screen_height = env.get_screen_res()
+        if install == '1':
+            # Install APK
+            apk_path = Path(settings.UPLD_DIR) / checksum / f'{checksum}.apk'
+            status, output = env.install_apk(
+                apk_path.as_posix(),
+                package,
+                reinstall)
+            if not status:
+                # Unset Proxy
+                env.unset_global_proxy()
+                msg = (f'This APK cannot be installed. Is this APK '
+                       f'compatible the Android VM/Emulator?\n{output}')
+                return print_n_send_error_response(
+                    request,
+                    msg,
+                    api)
+        logger.info('Testing Environment is Ready!')
+        context = {'screen_witdth': screen_width,
+                   'screen_height': screen_height,
+                   'package': package,
+                   'hash': checksum,
+                   'android_version': version,
+                   'version': settings.MOBSF_VER,
+                   'title': 'Dynamic Analyzer'}
+        template = 'dynamic_analysis/android/dynamic_analyzer.html'
+        if api:
+            return context
+        return render(request, template, context)
+    except Exception:
+        logger.exception('Dynamic Analyzer')
+        return print_n_send_error_response(
+            request,
+            'Dynamic Analysis Failed.',
+            api)
+
+
+def httptools_start(request):
+    """Start httprools UI."""
+    logger.info('Starting httptools Web UI')
+    try:
+        httptools_url = get_http_tools_url(request)
+        stop_httptools(httptools_url)
+        start_httptools_ui(settings.PROXY_PORT)
+        time.sleep(3)
+        logger.info('httptools UI started')
+        if request.GET['project']:
+            project = request.GET['project']
+        else:
+            project = ''
+        url = f'{httptools_url}/dashboard/{project}'
+        return HttpResponseRedirect(
+            url)  # lgtm [py/reflective-xss] lgtm [py/url-redirection]
+    except Exception:
+        logger.exception('Starting httptools Web UI')
+        err = 'Error Starting httptools UI'
+        return print_n_send_error_response(request, err)
+
+
+def logcat(request, api=False):
+    logger.info('Starting Logcat streaming')
+    try:
+        pkg = request.GET.get('package')
+        if pkg:
+            if not strict_package_check(pkg):
+                return print_n_send_error_response(
+                    request,
+                    'Invalid package name',
+                    api)
+            template = 'dynamic_analysis/android/logcat.html'
+            return render(request, template, {'package': pkg})
+        if api:
+            app_pkg = request.POST['package']
+        else:
+            app_pkg = request.GET.get('app_package')
+        if app_pkg:
+            if not strict_package_check(app_pkg):
+                return print_n_send_error_response(
+                    request,
+                    'Invalid package name',
+                    api)
+            adb = os.environ['MOBSF_ADB']
+            g = proc.Group()
+            g.run([adb, 'logcat', app_pkg + ':V', '*:*'])
+
+            def read_process():
+                while g.is_pending():
+                    lines = g.readlines()
+                    for _, line in lines:
+                        yield 'data:{}\n\n'.format(line)
+            return StreamingHttpResponse(read_process(),
+                                         content_type='text/event-stream')
+        return print_n_send_error_response(
+            request,
+            'Invalid parameters',
+            api)
+    except Exception:
+        logger.exception('Logcat Streaming')
+        err = 'Error in Logcat streaming'
+        return print_n_send_error_response(request, err, api)
+
+
+def trigger_static_analysis(request, checksum):
+    """On device APK Static Analysis."""
+    try:
+        identifier = None
+        if not is_md5(checksum):
+            return print_n_send_error_response(
+                request,
+                'Invalid MD5')
+        package = get_package_name(checksum)
+        if not package:
+            return print_n_send_error_response(
+                request,
+                'Cannot get package name from checksum')
+        try:
+            identifier = get_device()
+        except Exception:
+            pass
+        if not identifier:
+            err = 'Cannot connect to Android Runtime'
+            return print_n_send_error_response(request, err)
+        env = Environment(identifier)
+        apk_file = env.get_apk(checksum, package)
+        if not apk_file:
+            err = 'Failed to download APK file'
+            return print_n_send_error_response(request, err)
+        data = {
+            'analyzer': 'static_analyzer',
+            'status': 'success',
+            'hash': checksum,
+            'scan_type': 'apk',
+            'file_name': f'{package}.apk',
+        }
+        add_to_recent_scan(data)
+        return HttpResponseRedirect(
+            f'/static_analyzer/?name='  # lgtm [py/url-redirection]
+            f'{package}.apk&checksum={checksum}'
+            f'&type=apk')
+    except Exception:
+        msg = 'On device APK Static Analysis'
+        logger.exception(msg)
+        return print_n_send_error_response(request, msg)