grepsr-cli 0.7.4__py3-none-any.whl

grepsrcli/core/test_local.py

@@ -0,0 +1,136 @@
+from os import path, system
+import os
+import json
+import subprocess
+import platform
+from .message_log import Log
+from .config import load_config, config_path as CONFIG_PATH
+from .utils import get_docker_entrypoints, write_text, read_text
+
+class TestLocal:
+    def __init__(self, type, base_path, plugin_name, params=None, params_file=None, is_multi_proc=False):
+        self.type = type
+        self.plugin_name = plugin_name
+        self.base_path = base_path
+
+        self.tmp_path = path.expanduser('~') + '/.grepsr/tmp/'
+        self.config = load_config('config.yml')
+        self.type_config = self.config[type]
+        self.params = params
+        self.params_file = params_file
+        self.is_multi_proc = is_multi_proc
+        self.test_script()
+
+    def test_script(self):
+        if self.type == "php":
+            if "env" in self.type_config:
+                if self.type_config['env']:
+                    self.test_script_php(env=True)
+                else:
+                    self.test_script_php(env=False)
+            else:
+                self.test_script_php(env=False)
+        elif self.type == "php_next":
+            self.test_script_php_next()
+
+    def test_script_php(self, env=False):
+        if self.is_multi_proc is None:
+            # only use the value of multiprocess if it is not passed explicitly from cmd line
+            try:
+                self.is_multi_proc = '1' if self.type_config['multiprocess'] == True else '0'
+            except KeyError:
+                pass
+        env_vars = []
+        if env:
+            env_vars = [
+                f'-e {env_var}={self.type_config["env"][env_var]}' for env_var in self.type_config["env"].keys()
+            ]
+        if os.environ.get('GCLI_CHILD_PROC_NUM'):
+            print('\n', flush=True)
+            print('This is Child Process Number:', os.environ.get('GCLI_CHILD_PROC_NUM'), flush=True)
+            print('\n', flush=True)
+            env_vars.append(f'-e GCLI_CHILD_PROC_NUM={os.environ.get("GCLI_CHILD_PROC_NUM")}')
+        sdk_args_list = [
+            f'-s {self.plugin_name}',
+        ]
+        if self.is_multi_proc is not None:
+            sdk_args_list.append(f'-m {self.is_multi_proc}')
+        docker_image_name = self.type_config["sdk_image"]
+        json_content = self.params
+        pre_entrypoint = self.type_config.get('pre_entry_run_file')
+        only_service = (not self.params) and (not self.params_file) and (not pre_entrypoint)
+        bash_cmd = ''
+        if only_service:
+            sdk_args = ' '.join(sdk_args_list)
+        else:
+            if self.params_file:
+                # should be able to pass any path, not limited to just tmp/
+                fp = path.expandvars(path.expanduser((self.params_file)))
+                if not path.isabs(fp):
+                    fp = path.join(self.tmp_path, fp)
+                with open(fp, 'r', encoding='UTF-8') as fd:
+                    json_content = fd.read()
+
+            pipe_params = ''
+            sh_contents = ['#!/bin/bash']
+            if json_content:
+                sdk_args_list.append("-p")
+
+                # json decode and encode is required to remove spaces from the passed params
+                # because the bash shell file that forwards argument is not quoted ($@)
+                # which causes it to treat spaces anywhere as being a different argv
+                params_decoded = json.loads(json_content)
+                compact_json = json.dumps({'params': params_decoded}, separators=[',', ':'])
+                # now that all non important spaces are removed, replace spaces
+                # inside (keys and) values with unicode code-point which is luckily accepted by shell.
+                # hack: if the shell cannot handle space char, sneak the space with a makeup.
+                compact_json = compact_json.replace(' ', '\\u0020')
+                
+                tmp_json_file = '_gcli_internal_params_tmp.json'
+                # write the compacted json, to tmp/ directory whcih will get mounted 
+                # to docker container. Params are written to temporary file and not passed
+                # in the shell due to quoting issues.
+                write_text(path.join(self.tmp_path, tmp_json_file), compact_json)
+                pipe_params = f"$(</tmp/{tmp_json_file})"
+                
+            
+            custom_shell_file = '_gcli_internal_shell_file.sh'
+            custom_shell_file_local = path.join(self.tmp_path, custom_shell_file)
+            
+            if pre_entrypoint:
+                sh_contents.append(read_text(path.join(self.tmp_path, pre_entrypoint)))
+                
+            sdk_args = ' '.join(sdk_args_list)
+            original_entrypoints = get_docker_entrypoints(image_name=docker_image_name)
+            sh_contents.append(f'source {original_entrypoints[0]} {sdk_args} {pipe_params}')
+            write_text(custom_shell_file_local, '\n'.join(sh_contents))
+
+            bash_cmd = f'source /tmp/{custom_shell_file}'
+
+        win_single = (platform.system() == 'Windows') and (self.is_multi_proc == '1')
+        commands = [
+                f'docker run {"-i" if win_single else ""} -t --network="host" --rm',
+                f'-v {self.base_path}:/home/grepsr/vortex-plugins/{path.basename(self.base_path)}',
+                f'-v {self.tmp_path}:/tmp',
+                f'-e APP_ENV={self.config["app_env"]}',
+                " ".join(env_vars),
+                f'--entrypoint=""' if not only_service else '',
+                docker_image_name,
+                f'bash -ci "{bash_cmd}"' if not only_service else sdk_args,
+        ]
+
+        command = ' '.join([command for command in commands if command])
+        # print(command, flush=True)
+        system(command)
+
+    def test_script_php_next(self):
+
+        command = f'''docker run -t -i --network="host" --rm \
+                     -v {self.base_path}:/home/grepsr/vortex-backend-next/scraper-plugins \
+                     -v {self.tmp_path}:/tmp \
+                     -e APP_ENV={self.config['app_env']} \
+                     {self.type_config['sdk_image']} -s {self.plugin_name} '''
+        system(command)
+
+    def test_script_local(self):
+        pass

grepsrcli/core/utils.py

@@ -0,0 +1,308 @@
+import json
+import subprocess
+from pathlib import Path
+from os import listdir, path, mkdir
+import re
+from jinja2 import Template
+from .config import load_config
+from terminaltables import SingleTable
+from .message_log import Log
+
+
+def get_plugin_path(plugin_name, type='php', all_types=False):
+    """ returns the path of the plugin if plugin name exist
+
+    Args:
+        plugin_name (str): name of the service plugin .
+        type (str, optional): php|py|node. Defaults to 'php'.
+        all_types: retuns the path regardless the plugin types i.e php, py and node
+    """
+    config = load_config('config.yml')
+    if all_types == False:
+        if type == 'php':
+            if _generate_plugin_path(plugin_name, config['php']['paths']):
+                return _generate_plugin_path(plugin_name, config['php']['paths'])
+            else:
+                return False
+        elif type == 'node':
+            if _generate_plugin_path(plugin_name, config['node']['paths']):
+                return _generate_plugin_path(plugin_name, config['node']['paths'])
+            else:
+                return False
+        elif type == 'py':
+            if _generate_plugin_path(plugin_name, config['python']['paths']):
+                return _generate_plugin_path(plugin_name, config['python']['paths'])
+            else:
+                return False
+        elif type == 'php_next':
+            if _generate_plugin_path(plugin_name, config['php_next']['paths']):
+                return _generate_plugin_path(plugin_name, config['php_next']['paths'])
+            else:
+                return False
+        elif type == 'node_next':
+            if _generate_plugin_path(plugin_name, config['node_next']['paths']):
+                return _generate_plugin_path(plugin_name, config['node_next']['paths'])
+            else:
+                return False
+        else:
+            Log.error("Invalid params for type.")
+            return False
+    else:
+        # todo: do not hardcode this
+        base_paths_list = [
+            config['php']['paths'] if 'php' in config else None,
+            config['node']['paths'] if 'node' in config else None,
+            config['python']['paths'] if 'python' in config else None,
+            config['php_next']['paths'] if 'php_next' in config else None,
+            config['node_next']['paths'] if 'node_next' in config else None
+        ]
+
+        for base_paths in base_paths_list:
+            if base_paths is not None and _generate_plugin_path(plugin_name, base_paths):
+                return _generate_plugin_path(plugin_name, base_paths)
+        Log.error("Plugin name not found.")
+        return False
+
+
+def create_boilerplate(folder_path, boilerplate, data, extention,):
+    """ to create boilerplate for a given path
+
+    Args:
+        folder_path (str): the destination path
+        boilerplate (str): the name of the boilerplate
+        data (dict): the data to be rendered
+    """
+
+    service_name = path.basename(folder_path)
+    try:
+        mkdir(folder_path)
+    except OSError as err:
+        Log.error(err)
+        return
+
+    dest_path = '{}/{}.{}'.format(
+                folder_path, service_name, extention)
+
+    render_boilerplate(boilerplate=boilerplate, data=data,
+                       destination_path=dest_path)
+
+    Log.info(
+        'Plugin created at {}'.format(dest_path))
+
+
+def show_schema(base_path):
+    """ this method will show the schema of a plugin if it has schema.json
+
+    Args:
+        base_path (str): the base path of the plugin  eg: /home/vtx-services/aaa_com/
+    """
+
+    schema_path = base_path + '/schema.json'
+    if path.exists(schema_path):
+        try:
+            with open(schema_path, 'r') as f:
+                schema = f.read()
+                schema = json.loads(schema)
+                for page in schema.keys():
+                    Log.standout(f"Schema for Page: {page}")
+                    schema_heading = ['field', 'type', 'pattern']
+                    table_data = [
+                        schema_heading
+                    ]
+
+                    for th, td in schema[page]['schema']['properties'].items():
+                        if 'pattern' in td:
+                            row = [th, td['type'], td['pattern']]
+                        else:
+                            row = [th, td['type'], '']
+
+                        table_data.append(row)
+
+                    print(SingleTable(table_data).table)
+                return True
+        except:
+            Log.warn("Schema Structured Incorrectly")
+    else:
+        Log.warn("Schema Not Found")
+        return False
+
+COMMENT_BLOCK_PATTERN = re.compile(r"(\/\*\*.?\n)(.*?)(\s*\*\/\s*\n)", re.DOTALL)
+CLASS_DECLARATION_PATTERN = re.compile(r'class\s+\w+\s+extends\s(\w+)\b', re.DOTALL)
+
+def user_input(msg, default='Y'):
+    while True:
+        choice = input(msg + ' [Y/n]: ' if default == 'Y' else ' [y/N]: ').upper()
+        if choice == 'N':
+            return 'N'
+        elif choice == 'Y':
+            return 'Y'
+        elif choice == '':
+            return default
+
+def list_dependencies(plugin_name):
+    """list dependencies by listing all base_class"""
+
+    dependencies = set()
+    while True:
+        plugin_dir_path = get_plugin_path(plugin_name)
+        if not plugin_dir_path:
+            break
+        plugin_path = path.join(plugin_dir_path, plugin_name + '.php')
+        contents = read_text(plugin_path)
+        match = CLASS_DECLARATION_PATTERN.search(contents)
+        if not match:
+            break
+        base_class = match.group(1)
+        if base_class == 'Vtx_Service_Plugin':
+            break
+        plugin_name = base_class
+        dependencies.add(plugin_name)
+    return list(dependencies)
+
+def insert_all_chained_dependencies(plugin_name):
+    """add dependencies if the dependencies follow service_code/service_code.php pattern, however deep"""
+
+    dependencies = list_dependencies(plugin_name)
+    plugin_dir_path = get_plugin_path(plugin_name)
+    if not plugin_dir_path:
+        return
+    plugin_path = path.join(plugin_dir_path, plugin_name + '.php')
+    contents = read_text(plugin_path)
+    mappings = get_comment_block(contents)
+    if 'Dependencies' not in mappings:
+        mappings['Dependencies'] = ''
+        original_deps = []
+    else:
+        original_deps = mappings['Dependencies'].split(',')
+    
+    mappings['Dependencies'] = ','.join(list(set(dependencies + original_deps)))
+    contents = set_comment_block(contents, mappings)
+    write_text(plugin_path, contents)
+    return True
+
+def set_comment_block(script, mappings):
+    comment_block = ' * ' + '\n * '.join([f'{k}: {v}' for k, v in mappings.items()])
+    subbed = COMMENT_BLOCK_PATTERN.sub('\\1' + comment_block + '\\3', script)
+    return subbed
+
+def get_comment_block(script):
+    match = COMMENT_BLOCK_PATTERN.search(script)
+    if not match:
+        return {}
+    doc = match.group(2)
+    lines = doc.splitlines()
+    mappings = {}
+    for line in lines:
+        splitted = line.lstrip(' *').split(':', 1)
+        if len(splitted) != 2:
+            continue
+        k, v = splitted
+        mappings[k.strip()] = v.strip()
+    return mappings
+
+def get_plugin_info(plugin_path):
+    """ get plugin's info like service_name,pid,description from the plugin's base folder.
+    It does so by reading the file and looking at the info from plugin which is commented
+    at the beginning.
+
+    Args:
+        plugin_path (str): the path of the directory where we find the plugin.
+    """
+    plugin_file_path = listdir(plugin_path)
+
+    files = [f for f in plugin_file_path]
+    plugin_file = ''
+    for file in files:
+        if path.basename(plugin_path) in file:
+            plugin_file = file
+            break
+
+    if plugin_file:
+        try:
+            with open(path.join(plugin_path, plugin_file)) as f:
+                script = f.read()
+                match = CLASS_DECLARATION_PATTERN.search(script)
+                base_class = None
+                if match:
+                    base_class = match[1]
+
+                # TODO: use tree-sitter to parse php file.
+                mappings = {k.upper(): v for k, v in get_comment_block(script).items()}
+                if not mappings:
+                    return {}
+                dependencies_str = mappings.get('DEPENDENCIES', '')
+                dependencies = [d.strip() for d in dependencies_str.split(',')]
+                pid_line = mappings.get('PID', '')
+                ssv = pid_line.split(' ')
+                pid = ssv[0].strip()
+
+                pid_forced = False
+                if len(ssv) > 1:
+                    pid_forced = ssv[1].strip() == 'force'
+                
+                # TODO: to parse php import mechanisms like require_once, include_once, etc. for dependencies 
+                return {
+                    'pid': pid,
+                    'pid_forced': pid_forced,
+                    'report_name': mappings.get('NAME'),
+                    'description': mappings.get('DESCRIPTION'),
+                    'dependencies': dependencies,
+                    'base_class': base_class
+                }
+        except KeyError as e:
+            pass
+    return {}
+
+def render_boilerplate(boilerplate, data, destination_path):
+    """parse boilerplate from template directory to start a project
+
+    Args:
+        boilerplate (str): name of boilerplate template
+        data (dict): input for the boilerplate
+        destination_path: the final path where the final content needs to be saved
+    """
+
+    template_dir = Path(__file__).parent.parent.absolute()
+    template_file = '{}/templates/{}'.format(template_dir, boilerplate)
+    with open(template_file) as file:
+        template = Template(file.read())
+        with open(destination_path, 'w') as dest_file:
+            dest_file.write(template.render(data))
+
+
+def _generate_plugin_path(plugin_name, paths):
+    
+    for service_path in paths:
+        plugin_path = path.join(service_path, plugin_name)
+        plugin_path = path.expanduser(plugin_path)
+        if path.exists(plugin_path):
+            return plugin_path
+    return False
+
+
+def get_docker_entrypoints(image_name):
+    o = subprocess.run(f'docker inspect {image_name}', shell=True, capture_output=True, text=True)
+    try:
+        image_info = json.loads(o.stdout)
+    except json.JSONDecodeError:
+        raise Exception('Running docker inspect failed. Is SDK image correct?')
+    try:
+        return image_info[0]['Config']['Entrypoint']
+    except IndexError:
+        raise Exception('Docker inspect failed. Is Docker Running? Is SDK image available?')
+
+def write_text(file_path, contents):
+    "helper function to write text to file with compatible encoding and no newline magic"
+
+    # turn off universal-newline mode for less surprises.
+    with open(file_path, 'w', encoding='UTF-8', newline='') as fd:
+        fd.write(contents)
+
+def read_text(file_path):
+    "helper function to read text from file with compatible encoding and no newline magic"
+
+    # turn off universal-newline mode for less surprises.
+    with open(file_path, 'r', encoding='UTF-8', newline='') as fd:
+        contents = fd.read()
+
+    return contents

grepsrcli/main.py

@@ -0,0 +1,91 @@
+from cement import App, TestApp, init_defaults
+from cement.core.exc import CaughtSignal
+from .core.exc import GrepsrCliError
+from .controllers.base import Base
+from .controllers.crawler import Crawler, CrawlerBase
+from .controllers.report import Report, ReportBase
+from .controllers.generate import Generate, GenerateBase
+
+# configuration defaults
+CONFIG = init_defaults('grepsrcli')
+
+
+class GrepsrCli(App):
+    """Grepsr Cli primary application."""
+
+    class Meta:
+        label = 'gcli'
+
+        # configuration defaults
+        config_defaults = CONFIG
+
+        # call sys.exit() on close
+        exit_on_close = True
+
+        # load additional framework extensions
+        extensions = [
+            'yaml',
+            'colorlog',
+            'jinja2',
+        ]
+
+        # configuration handler
+        config_handler = 'yaml'
+
+        # configuration file suffix
+        config_file_suffix = '.yml'
+
+        # set the log handler
+        log_handler = 'colorlog'
+
+        # set the output handler
+        output_handler = 'jinja2'
+
+        # register handlers
+        handlers = [
+            Base,
+            CrawlerBase,
+            Crawler,
+            ReportBase,
+            Report,
+            GenerateBase,
+            Generate
+        ]
+
+
+class GrepsrCliTest(TestApp, GrepsrCli):
+    """A sub-class of GrepsrCli that is better suited for testing."""
+
+    class Meta:
+        label = 'grepsrcli'
+
+
+def main():
+    with GrepsrCli() as app:
+        try:
+            app.run()
+
+        except AssertionError as e:
+            print('AssertionError > %s' % e.args[0])
+            app.exit_code = 1
+
+            if app.debug is True:
+                import traceback
+                traceback.print_exc()
+
+        except GrepsrCliError as e:
+            print('GrepsrCliError > %s' % e.args[0])
+            app.exit_code = 1
+
+            if app.debug is True:
+                import traceback
+                traceback.print_exc()
+
+        except CaughtSignal as e:
+            # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error)
+            print('\n%s' % e, flush=True)
+            app.exit_code = 0
+
+
+if __name__ == '__main__':
+    main()

grepsrcli/templates/autocomplete.jinja2

@@ -0,0 +1,12 @@
+#!/bin/bash
+{% for p in paths %}
+f{{p.ind}}=$(find {{p.dest_path}} -mindepth 1 -type d -printf "%f ")
+{% endfor %}
+
+_gcliComplete()
+{
+    local cur=${COMP_WORDS[COMP_CWORD]}
+    COMPREPLY=( $(compgen -W "{% for p in paths %} $f{{p.ind}} {% endfor %}" -- $cur) )
+}
+
+complete -F _gcliComplete gcli

grepsrcli/templates/autocomplete_zsh.jinja2

@@ -0,0 +1,20 @@
+#!/bin/zsh
+
+# install findutils if not already installed
+if brew ls --versions findutils > /dev/null; then echo "installed" > /dev/null; else brew install findutils; fi
+
+# load the modules for bash autompletion
+autoload -U +X compinit && compinit
+autoload -U +X bashcompinit && bashcompinit
+
+{% for p in paths %}
+f{{p.ind}}=$(gfind {{p.dest_path}} -mindepth 1 -type d -printf "%f ")
+{% endfor %}
+
+_gcliComplete()
+{
+    local cur=${COMP_WORDS[COMP_CWORD]}
+    COMPREPLY=( $(compgen -W "{% for p in paths %} $f{{p.ind}} {% endfor %}" -- $cur) )
+}
+
+complete -F _gcliComplete gcli

grepsrcli/templates/composer.jinja2

@@ -0,0 +1,21 @@
+{
+    "repositories": [{
+        "type": "package",
+        "package": {
+            "name": "{{plugin_name}}/{{plugin_name}}",
+            "version": "{{version}}",
+            "dist": {
+                "url": "{{presigned_url}}",
+                "type": "tar"
+            },
+            "autoload": {
+                "classmap": [
+                    "/"
+                ]
+            }
+        }
+    }],
+    "require": {
+        "{{plugin_name}}/{{plugin_name}}": "{{version}}"
+    }
+}