grepsr-cli 0.7.4__py3-none-any.whl

grepsrcli/controllers/generate.py

@@ -0,0 +1,34 @@
+from typing import final
+from cement import Controller, ex
+import itertools;
+from ..core.config import load_config, generate_autocomplete_script
+
+class GenerateBase(Controller):
+    class Meta:
+        label = 'generate_base'
+
+
+class Generate(Controller):
+    config = load_config('config.yml')
+
+    class Meta:
+        label = 'generate'
+        stacked_on = 'generate_base'
+        stacked_type = 'nested'
+        
+    @ex(
+        help="Make and integrate tab autocompletion script",
+    )
+    def tab_completion(self):
+        config = self.config
+        
+        base_paths_list = [
+            config['php']['paths'] if 'php' in config else [],
+            config['node']['paths'] if 'node' in config else [],
+            config['python']['paths'] if 'python' in config else [],
+            config['php_next']['paths'] if 'php_next' in config else []
+        ]
+               
+        final_list = list(itertools.chain.from_iterable(base_paths_list))
+           
+        generate_autocomplete_script(final_list)

grepsrcli/controllers/report.py

@@ -0,0 +1,63 @@
+from cement import Controller, ex
+from webbrowser import open_new_tab
+from ..core.config import load_config
+from ..core.utils import get_plugin_path, get_plugin_info
+from ..core.report_api import get_report_from_pid, run_report_by_rid
+
+
+class ReportBase(Controller):
+    class Meta:
+        label = 'report_base'
+
+
+class Report(Controller):
+    config = load_config('config.yml')
+
+    class Meta:
+        label = 'report'
+        stacked_on = 'report_base'
+        stacked_type = 'nested'
+
+    @ex(
+        help="run report to live",
+        arguments=[
+            (['-rid', '--report-id'], {
+                "action": "store",
+                "dest": "report_id"
+            }),
+            (['-s', '--service'], {
+                "action": "store",
+                "dest": "service_code"
+            })
+        ]
+    )
+    def run(self):
+
+        if(self.app.pargs.service_code is not None):
+            service_code = self.app.pargs.service_code
+            plugin_info = get_plugin_info(
+                get_plugin_path(service_code, all_types=True))
+
+            rid = get_report_from_pid(
+                plugin_info['pid'], plugin_info['report_name'])
+
+            if(rid):
+                run_api_message = run_report_by_rid(rid)
+                self.app.log.info("Report: {}, message: {}".format(
+                    service_code, run_api_message))
+                open_new_tab(
+                    f'https://appnext.grepsr.com/projects/{plugin_info["pid"]}')
+            else:
+                self.app.log.warning(
+                    "We couldn't run report via report name, please try running it manually via the url")
+
+                self.app.log.info(
+                    f'App Url: https://appnext.grepsr.com/projects/{plugin_info["pid"]}')
+
+        elif (self.app.pargs.report_id is not None):
+            report_id = self.app.pargs.report_id
+            run_api_message = run_report_by_rid(report_id)
+            self.app.log.info("Report ID: {}, message: {}".format(
+                report_id, run_api_message))
+        else:
+            self.app.log.error("please use a valid syntax")

grepsrcli/core/aws_s3.py

@@ -0,0 +1,39 @@
+from base64 import b64decode
+from boto3.session import Session
+
+class S3():
+
+    def __init__(self, aws_id,  aws_sec_key):
+        self.aws_id = aws_id
+        self.secret_key = aws_sec_key
+        
+        self.aws_session = Session(
+            region_name='eu-central-1',
+            aws_access_key_id=self.aws_id,
+            aws_secret_access_key=self.secret_key
+        )
+
+    def get_decoded_auth_creds(self):
+        resp = self.aws_session.client('ecr').get_authorization_token()
+        encoded = resp.get('authorizationData')[0].get('authorizationToken')
+        return b64decode(encoded).decode('UTF-8')
+
+    def get_secure_url(self, bucket, filename, expires_in=7200, insecure=False):
+                
+        # max is 7 days
+        if not expires_in:
+            expires_in = 7200
+        elif expires_in > 604800:
+            expires_in = 604800
+        
+        url = self.aws_session.client('s3').generate_presigned_url(
+            'get_object',
+            Params={
+                'Bucket': bucket, 'Key': filename},
+            ExpiresIn=expires_in
+        )
+        
+        if insecure:
+            return url.split("?")[0]
+        else:
+            return url

grepsrcli/core/config.py

@@ -0,0 +1,191 @@
+import yaml
+from os import path, remove, system
+from pathlib import Path
+from .message_log import Log
+from jinja2 import Template
+import platform
+
+
+config_path = path.expanduser('~/.grepsr')
+
+
+def load_config(file_name, file_path=None):
+    """Loads yml config file for user configuration
+
+    Args:
+        file_name (str): the name of file in the path gcli/config
+    """
+
+    if file_path is None:
+        file_path = config_path
+    try:
+        file = f'{file_path}/{file_name}'
+        with open(file, 'r') as f:
+            file_content = f.read()
+            return yaml.full_load(file_content)
+    except:
+        Log.error("{}  not found in Path {}".format(file_name, file_path))
+
+
+def save_config(update=False):
+
+    if update == False:
+        setup_config = {}
+    else:
+        setup_config = load_config('config.yml')
+
+    platforms = ['php', 'node', 'python', 'php_next']
+    crawler_paths = []
+
+    for plat in platforms:
+        if yes_no_prompt(f"\nProcess configuration for {Log.OKCYAN + plat + Log.ENDC} platform?"):
+            if update == False:
+                setup_config[plat] = {}
+                setup_config[plat]['paths'] = []
+
+                if(plat == 'php'):
+                    setup_config[plat]['sdk_image'] = '486559570021.dkr.ecr.eu-central-1.amazonaws.com/vortex/sdk:stable'
+                elif(plat == 'php_next'):
+                    setup_config[plat]['sdk_image'] = '486559570021.dkr.ecr.eu-central-1.amazonaws.com/vortex/backend-next:latest'
+                else:
+                    setup_config[plat]['sdk_image'] = ''
+
+            setup_config[plat]['env'] = {}
+            Log.heading("\nStep 1: Path Configuration")
+
+            if update == True:
+                if yes_no_prompt("Do you want to update crawler paths?") == True:
+                    setup_config[plat]['paths'] = []
+                    while True:
+                        base_path = prompt_for(
+                            prompt="Enter path for your plugin directory: ")
+                        base_path = path.expanduser(base_path.rstrip('/'))
+                        setup_config[plat]['paths'].append(base_path)
+                        crawler_paths.append(base_path)
+                        if(yes_no_prompt("Do you want to another path?") == False):
+                            break
+            elif update == False:
+                while True:
+                    base_path = prompt_for(
+                        prompt="Enter path for your plugin directory: ")
+                    base_path = path.expanduser(base_path.rstrip('/'))
+                    setup_config[plat]['paths'].append(base_path)
+                    crawler_paths.append(base_path)
+                    if(yes_no_prompt("Do you want to another path?") == False):
+                        break
+            else:
+                pass
+
+            Log.heading("\nStep 2: SDK Configuration")
+            if update == True:
+                if yes_no_prompt("Do you want to update SDK image ?") == True:
+                    if plat == 'php':
+                        setup_config[plat]['sdk_image'] = '486559570021.dkr.ecr.eu-central-1.amazonaws.com/vortex/sdk:stable'
+                    elif plat == 'php_next':
+                        setup_config[plat]['sdk_image'] = '486559570021.dkr.ecr.eu-central-1.amazonaws.com/vortex/backend-next:latest'
+                    else:
+                        setup_config[plat]['sdk_image'] = ''
+
+                    sdk_image = prompt_for(
+                        prompt=f"Enter SDK image for {Log.OKCYAN + plat + Log.ENDC} (defaults to stable sdk if empty): ", empty=True)
+                    if sdk_image is not None:
+                        setup_config[plat]['sdk_image'] = sdk_image
+            elif update == False:
+                sdk_image = prompt_for(
+                    prompt=f"Enter SDK image for {Log.OKCYAN + plat + Log.ENDC} (defaults to stable sdk if empty): ", empty=True)
+                if sdk_image is not None:
+                    setup_config[plat]['sdk_image'] = sdk_image
+            else:
+                pass
+
+            Log.heading("\nStep 3: Environment variable Coniguration")
+            if yes_no_prompt("Do you want to add environment variables?") == True:
+                while True:
+                    env_key_val = prompt_for(
+                        f"Enter env variable for {Log.OKCYAN + plat + Log.ENDC} (Eg:- CFG_PARAMS_SELENIUM_HOST: 192.168.90.64) \n")
+                    try:
+                        env_key_val = env_key_val.split(':')
+                        setup_config[plat]['env'][env_key_val[0].strip()
+                                                  ] = env_key_val[1].strip()
+                    except:
+                        Log.error("Please follow the correct format")
+                    if yes_no_prompt("Do you want to another env variable?") == False:
+                        break
+
+    if update == True:
+        if yes_no_prompt("Do you want to update your aws keys?") == True:
+            setup_config['aws_access_key_id'] = prompt_for(
+                f"Enter aws id \n")
+            setup_config['aws_secret_access_key'] = prompt_for(
+                f"Enter aws secret key \n")
+    else:
+        setup_config['aws_access_key_id'] = prompt_for(
+            f"Enter aws id \n")
+        setup_config['aws_secret_access_key'] = prompt_for(
+            f"Enter aws secret key \n")
+
+    setup_config['app_env'] = 'tst'
+
+    with open(config_path + '/config.yml', 'w') as w:
+        w.write(yaml.dump(setup_config))
+
+    generate_autocomplete_script(crawler_paths)
+
+def prompt_for(prompt, empty=True):
+    while True:
+        ans = input(prompt)
+        if not empty and ans == '':
+            continue
+        break
+        
+    return ans
+
+def yes_no_prompt(ques):
+    answer = input(f'{ques} (y/n) ')
+    if(answer in ['y', '']):
+        return True
+    else:
+        return False
+
+
+def generate_autocomplete_script(crawler_paths):
+    
+    data = {}
+    data['paths'] = []
+    for p in range(len(crawler_paths)):
+        d = {}
+        d['ind'] = p
+        d['dest_path'] = crawler_paths[p]
+        data['paths'].append(d)
+
+    template_dir = Path(__file__).parent.parent.absolute()
+
+    # generate autocomplete file for zsh and bash
+    for auto_script in ['autocomplete',  'autocomplete_zsh']:
+
+        if path.exists(config_path + f'/{auto_script}.sh'):
+            remove(config_path +
+                   f'/{auto_script}.sh')
+
+        template_file = '{}/templates/{}'.format(
+            template_dir, f'{auto_script}.jinja2')
+
+        with open(template_file) as file:
+            template = Template(file.read())
+
+        with open(config_path + f'/{auto_script}.sh', 'w') as dest_file:
+            dest_file.write(template.render(data))
+    Log.info(f"Generating Autocomplete Script for {platform.system()}")
+    if platform.system() == 'Linux':
+        
+        with open(path.expanduser('~/.bashrc')) as rc_file:
+            rc_file = rc_file.read()
+            if ".grepsr/autocomplete.sh" not in rc_file:
+                system(
+                    '''echo "source $HOME/.grepsr/autocomplete.sh" >>  ~/.bashrc''')
+    elif platform.system() == 'Darwin':
+        with open(path.expanduser(path.expanduser('~/.zshrc'))) as rc_file:
+            rc_file = rc_file.read()
+            if ".grepsr/autocomplete_zsh.sh" not in rc_file:
+                system(
+                    '''echo "source $HOME/.grepsr/autocomplete_zsh.sh" >>  ~/.zshrc''')

grepsrcli/core/exc.py

@@ -0,0 +1,4 @@
+
+class GrepsrCliError(Exception):
+    """Generic errors."""
+    pass

grepsrcli/core/message_log.py

@@ -0,0 +1,34 @@
+class Log:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+    @staticmethod
+    def info(message):
+        print(f'{Log.OKGREEN}INFO: {message}{Log.ENDC}')
+
+    @staticmethod
+    def error(message):
+        print(f'{Log.BOLD}{Log.FAIL}ERR: {message}{Log.ENDC}')
+
+    @staticmethod
+    def standout(message):
+        print(f'{Log.HEADER}{message}{Log.ENDC}')
+
+    @staticmethod
+    def success(message):
+        print(f'{Log.OKGREEN}SUCCESS: {message}{Log.ENDC}')
+
+    @staticmethod
+    def warn(message):
+        print(f'{Log.WARNING}WARNING: {message}{Log.ENDC}')
+
+    @staticmethod
+    def heading(message):
+        print(f'{Log.OKBLUE}{message}{Log.ENDC}')

grepsrcli/core/multiproc_server.py

@@ -0,0 +1,126 @@
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import json
+import os
+import subprocess
+import time
+import platform
+
+PORT = 8848
+PROC_TITLE = 'gcli_multiproc_procnum'
+CHILD_PROC_ENV_IDENTIFIER = 'GCLI_CHILD_PROC_NUM'
+PROCS = {}
+
+""" 
+def kill(proc_title, proc):
+    proc_pid = proc.pid
+    if platform.system() == 'Windows':
+        subprocess.run(
+            f'TASKKILL /F /T /FI "WINDOWTITLE eq {proc_title}*"', shell=True)
+    elif platform.system() == 'Linux':
+        os.killpg(proc_pid, signal.SIGINT)
+ """
+
+
+def spawn_gcli_terminal(proc_num, service_code, params):
+    params_str = json.dumps(params, separators=[',', ':'])
+    '''
+    win_title = f'{PROC_TITLE}_{proc_num}'
+    if platform.system() == 'Windows':
+        cmd = f'start "{win_title}" cmd /k "sleep 1 && echo {service_code} && sleep 15 && echo {params_str}"'
+        proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        PROCS[win_title] = proc
+    elif platform.system() == 'Linux':
+    '''
+
+    win_title = f'{proc_num}.gcli'
+    output_logs_dir = os.path.join(os.path.expanduser('~'), '.grepsr-experiments', 'multi_proc')
+    os.makedirs(output_logs_dir, exist_ok=True)
+    # turn multiprocess flag off to prevent infinite recursion.
+    cmds = ["gcli", "crawler", "test", "-m", "0", "-s", service_code, "-p", params_str]
+    # if platform.system() == 'Windows':
+    #     cmds[0] = 'C:\\Users\\DELL\\bin\\gcli.cmd'
+    with open(os.path.join(output_logs_dir, f'{win_title}.mp.out'), 'w') as fd:
+        # this env var tells that the current crawler is a child process initiated by main parent.
+        # this information might be needed somewhere
+        os.environ[CHILD_PROC_ENV_IDENTIFIER] = str(proc_num)
+        proc = subprocess.Popen(cmds, stdin=subprocess.PIPE, stdout=fd, stderr=subprocess.STDOUT)
+        # wait a little so that there is less chance of race condition when gcli starts processing.
+        time.sleep(2)
+        PROCS[win_title] = proc
+
+
+class ProcessHandler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = None
+        respo = {'success': False, 'message': ''}
+        try:
+            content_length = int(self.headers['content-length'])
+        except TypeError:
+            respo = {'success': False,
+                     'message': 'Could not convert content-length header to integer'}
+        if content_length is not None:
+            if self.headers['content-type'].split(';')[0] == 'application/json':
+                body = self.rfile.read(content_length).decode('UTF-8')
+                try:
+                    data = json.loads(body)
+                    service_code = data.get('service_code')
+                    params = data.get('params')
+                    proc_num = data.get('proc_num')
+                    if (service_code is None) or (params is None) or (proc_num is None):
+                        respo = {
+                            'success': False, 'message': 'No service code or params or proc number found'}
+                    else:
+                        spawn_gcli_terminal(proc_num, service_code, params)
+                        respo = {'success': True, 'data': {'message': 'Spawned a process successfully!'}}
+                except json.JSONDecodeError:
+                    respo = {'success': False,
+                             'message': 'request body is not JSON.'}
+
+        self.send_response(200)
+        self.send_header('content-type', 'application/json')
+        self.end_headers()
+
+        self.wfile.write(bytes(json.dumps(respo), "utf8"))
+
+def start():
+    with HTTPServer(('127.0.0.1', PORT), ProcessHandler) as server:
+        try:
+            print('C&C: MultiProcess facillitation Python Server started', flush=True)
+            server.serve_forever()
+        except (Exception, KeyboardInterrupt) as e:
+            if not isinstance(e, KeyboardInterrupt):
+                print('\n', flush=True)
+                print(f'Do not Panic. Caught Exception. `{e}`', flush=True)
+            # print('', flush=True)
+            # print('Force Exit. Cleaning server and child processes.', flush=True)
+            server.server_close()
+
+    for i, proc in PROCS.items():
+        # print('Terminating Child process:: ', i, flush=True)
+        proc.terminate()
+        proc.communicate()
+        proc.kill()
+        print('Terminated Child process:: ', i, flush=True)
+        # kill(i, proc)
+
+
+    # outs, errs = proc.communicate(signal.CTRL_BREAK_EVENT)
+    # outs, errs = proc.communicate(signal.CTRL_C_EVENT)
+
+    # if hasattr(signal, 'CTRL_C_EVENT'):
+    #     # windows. Need CTRL_C_EVENT to raise the signal in the whole process group
+    #     print('using win method to kill whole process group', flush=True)
+    #     os.kill(os.getpid(), signal.CTRL_C_EVENT)
+    #     # os.kill(os.getpid(), signal.CTRL_BREAK_EVENT)
+    # else:
+    #     print('using unix method to kill whole process group', flush=True)
+    #     # unix.
+    #     pgid = os.getpgid(os.getpid())
+    #     if pgid == 1:
+    #         os.kill(os.getpid(), signal.SIGINT)
+    #     else:
+    #         os.killpg(os.getpgid(os.getpid()), signal.SIGINT)
+
+if __name__ == '__main__':
+    pass

grepsrcli/core/report_api.py

@@ -0,0 +1,39 @@
+from ..core.config import load_config
+import requests as req
+import json
+
+
+config = load_config('config.yml')
+
+
+def get_report_from_pid(pid, report_name):
+    """[summary]
+
+    Args:
+        pid ([type]): [description]
+        report_id ([type]): [description]
+    """
+
+    res = req.get(
+        'https://api.grepsr.com/v1/report/list?project_id={}&x-api-key={}'.format(pid, config['api_key']))
+
+    res_json = json.loads(res.text)
+    reports = res_json['payload']
+    for report in reports:
+        if report['name'].strip() == report_name.strip():
+            return report['report_id']
+
+    return False
+
+
+def run_report_by_rid(rid):
+    """[summary]
+
+    Args:
+        rid ([type]): [description]
+    """
+
+    run_api = req.get(
+        'https://api.grep.sr/v1/report/run?report_id={}&x-api-key={}'.format(rid, config['api_key']))
+
+    return json.loads(run_api.text)['message']

grepsrcli/core/sdk_setup.py

@@ -0,0 +1,84 @@
+from os import path
+import os
+import shutil
+import subprocess
+import yaml
+from .message_log import Log
+from .config import load_config
+from .aws_s3 import S3
+
+
+class SDKSetup:
+    config = load_config('config.yml')
+    home_path = path.expanduser('~')
+    config_path = path.expanduser('~/.grepsr/config.yml')
+
+    def __init__(self, type, dryrun, sdk_image):
+        self.type_dict = self.config[type]
+        self.type_name = type
+        self.dryrun = dryrun
+        self.sdk_image = sdk_image
+
+        self.do_setup()
+
+    def check_path(self):
+        for service_dir in self.type_dict.get('paths') or []:
+            service_dir = path.expanduser(service_dir)
+            if not path.exists(service_dir):
+                Log.error(
+                    f'Directory {service_dir} was not found')
+                Log.error(
+                    "Please check ~/.grepsr/config.yml to check if paths are correctly added")
+                return False
+        return True
+
+    def generate_autocomplete_directory(self, docker_id, dir_name):
+        for service_dir in self.type_dict.get('paths') or []:
+            Log.info(
+                f"Adding autocomplete modules in {service_dir}. This could take some time...")
+            
+            autocomplete_dir = path.join(service_dir, '.autocomplete')
+            if os.path.exists(autocomplete_dir):
+                shutil.rmtree(autocomplete_dir)
+            os.makedirs(autocomplete_dir, exist_ok=True)
+
+            command = f"""docker cp -L {docker_id}:/home/grepsr/{dir_name}/lib {autocomplete_dir} && docker cp -L {docker_id}:/home/grepsr/{dir_name}/vendor {autocomplete_dir}
+            """
+            os.system(command)
+
+    def do_setup(self):
+        if self.sdk_image is not None:
+            with open(self.config_path, 'w') as w:
+                self.config[self.type_name]['sdk_image'] = self.sdk_image
+                w.write(yaml.dump(self.config))
+
+        if not self.check_path():
+            return False
+        if not self.dryrun:
+            Log.info("Logging in to AWS service")
+
+            s3 = S3(self.config['aws_access_key_id'], self.config['aws_secret_access_key'])
+            username, auth_token = s3.get_decoded_auth_creds().split(':')
+            sdk_image = self.sdk_image or self.type_dict['sdk_image']
+            hostname = sdk_image.split('/', 2)[0]
+            out = subprocess.run([
+                'docker', 'login', '--username', username, '--password-stdin', hostname
+            ], input=auth_token, text=True, stdout=subprocess.PIPE)
+            Log.info(out.stdout)
+
+            Log.info("Getting the SDK")
+            os.system(f"docker pull {sdk_image}")
+
+        docker_id = subprocess.run(
+            ['docker',  'create', self.type_dict['sdk_image']], stdout=subprocess.PIPE).stdout
+        docker_id = docker_id.decode('utf-8').rstrip()
+
+        if self.type_name == 'php':
+            self.generate_autocomplete_directory(
+                docker_id, 'vortex-backend')
+        elif self.type_name == 'php_next':
+            self.generate_autocomplete_directory(
+                docker_id, 'vortex-backend-next')
+
+        os.system(f'docker rm {docker_id}')
+        Log.info("SDK has been installed successfully")