pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/validators/validate_app_version.py

@@ -1,183 +0,0 @@
-import io
-import re
-from zipfile import ZipFile
-
-from biolib.validators import validator_utils  # type: ignore
-
-
-def validate_app_version(yaml_data, yaml_version, zip_file):
-    files = [filename.strip('/') for filename in zip_file.namelist()]
-
-    error_dict = {}
-    validate_unsupported_root_level_fields(yaml_data, error_dict, yaml_version)
-    validate_output_type(yaml_data, error_dict)
-    validate_consumes_stdin(yaml_data, error_dict)
-    validate_remote_hosts(yaml_data, error_dict)
-    validate_description_file(yaml_data, files, zip_file, error_dict)
-    validate_license_file(yaml_data, files, zip_file, error_dict)
-    return error_dict
-
-
-def validate_output_type(yaml_data, error_dict):
-    # output_type is not required
-    if 'output_type' in yaml_data.keys():
-        output_type = yaml_data['output_type']
-        stdout_render_types_choices = [type_tuple[0] for type_tuple in validator_utils.stdout_render_types]
-        if output_type not in stdout_render_types_choices:
-            error_dict['output_type'] = [
-                f'Invalid output_type specified for your app. output_type can be one of {stdout_render_types_choices}'
-            ]
-
-
-def validate_consumes_stdin(yaml_data, error_dict):
-    if 'consumes_stdin' in yaml_data.keys():
-        consumes_stdin = yaml_data['consumes_stdin']
-        if not isinstance(consumes_stdin, bool):
-            error_dict['consumes_stdin'] = [
-                f'Invalid consumes_stdin specified for your app. consumes_stdin can be true or false'
-            ]
-
-
-def validate_remote_hosts(yaml_data, error_dict):
-    if 'remote_hosts' in yaml_data.keys():
-        for hostname in yaml_data['remote_hosts']:
-            # No error message is returned if the hostname is valid
-            hostname_error_message = validator_utils.validate_hostname_and_return_error_message(hostname)
-            if hostname_error_message:
-                error_dict['remote_hosts'] = hostname_error_message
-
-
-def validate_description_file(yaml_data, files, zip_file: ZipFile, error_dict):
-    # Only check existence of description file if user specified it
-    if 'description_file' in yaml_data.keys():
-        description_path = yaml_data['description_file']
-
-        if description_path not in files:
-            error_dict['description_file'] = [
-                f'Could not find description file at {description_path}. \
-Please provide a path pointing to a markdown (.md) file'
-            ]
-        return
-
-    else:
-        description_path = 'README.md'
-
-    if description_path in files:
-        # TODO: Discuss this limit, seems very high :sweat:
-        # Check if description file is bigger than 100MB, written as 100000000 bytes
-        if zip_file.getinfo(description_path).file_size > 100000000:
-            error_dict['description_file'] = [f'The description file {description_path} must be less than 100MB']
-
-        validate_description_images(zip_file.open(description_path), files, zip_file, error_dict)
-
-
-def validate_license_file(yaml_data, files, zip_file, error_dict):
-    if 'license_file' in yaml_data.keys():
-        license_path = yaml_data['license_file']
-
-        if license_path not in files:
-            error_dict['license_file'] = [
-                f'Could not find license file at {license_path}. Please provide a path pointing to a license file'
-            ]
-            return
-
-    else:
-        license_path = 'LICENSE'
-
-    if license_path in files:
-        # Check if license file is bigger than 100MB, written as 100000000 bytes
-        if zip_file.getinfo(license_path).file_size > 100000000:
-            error_dict['license_file'] = [f'The license file {license_path} must be less than 100MB']
-
-
-def validate_is_open_source(yaml_data, error_dict):
-    if 'is_open_source' in yaml_data.keys():
-        if not isinstance(yaml_data['is_open_source'], bool):
-            error_dict['is_open_source'] = [
-                f'Invalid is_open_source specified for your app. is_open_source can be true or false'
-            ]
-
-
-supported_root_level_fields_base = [
-    'arguments',
-    'biolib_version',
-    'citation',
-    'consumes_stdin',
-    'description_file',
-    'license_file',
-    'modules',
-    'output_type',
-    'remote_hosts',
-]
-
-supported_root_level_fields_v1 = [
-    'client_side_include',
-]
-
-supported_root_level_fields_v2 = [
-    'source_files_ignore',
-]
-
-
-def validate_unsupported_root_level_fields(yaml_data, error_dict, yaml_version):
-    if yaml_version == 1:
-        supported_fields = supported_root_level_fields_base + supported_root_level_fields_v1
-    else:
-        supported_fields = supported_root_level_fields_base + supported_root_level_fields_v2
-
-    errors = []
-    for field in yaml_data.keys():
-        if field not in supported_fields:
-            errors.append(
-                f'The field {field} is not valid'
-            )
-
-    if errors:
-        error_dict['unsupported_fields'] = errors
-
-
-def validate_description_images(description_path, files, zip_file, error_dict):
-    REGEX_MARKDOWN_INLINE_IMAGE = re.compile(r'!\[(?P<alt>.*)\]\((?P<src>.*)\)')
-    image_filesize_limit_in_bytes = 5000000
-    supported_image_file_types = ('png', 'gif', 'jpg', 'jpeg')
-
-    description_images = {}
-    description_markdown = io.TextIOWrapper(description_path).read()
-
-    for img_alt, img_src_path in re.findall(REGEX_MARKDOWN_INLINE_IMAGE, description_markdown):
-
-        if img_src_path in description_images:
-            continue
-
-        if re.match(r'data:.*;base64,', img_src_path):
-            error_dict['description_file'] = [
-                'The Markdown description does not support base64 images. '
-                'Please specify images using their path in the application files: '
-                '![Example Alt Text](path/to/image.png)'
-            ]
-            return error_dict
-
-        if img_src_path not in files:
-            if len(img_src_path) > 200:
-                img_src_path = img_src_path[:200] + '...'
-            error_dict['description_file'] = [
-                f'In the Markdown description the image path {img_src_path} does not exist in application files'
-            ]
-            return error_dict
-
-        extension = img_src_path.split('.')[-1] if '.' in img_src_path else 'png'
-        if extension not in supported_image_file_types:
-            error_dict['description_file'] = [
-                f'In the Markdown description, the image {img_src_path} '
-                f'must point to an image of the following types {supported_image_file_types}.'
-            ]
-            return error_dict
-
-        # Limit image size
-        if zip_file.getinfo(img_src_path).file_size > image_filesize_limit_in_bytes:
-            error_dict['description_file'] = [
-                f'In the Markdown description, the image {img_src_path} is over '
-                f'{image_filesize_limit_in_bytes / 1000000} MB which is too large.'
-            ]
-
-        return error_dict

biolib/validators/validate_argument.py

@@ -1,134 +0,0 @@
-render_types = [
-    ('dropdown', 'Dropdown'),
-    ('file', 'File'),
-    ('hidden', 'Hidden'),
-    ('number', 'Number'),
-    ('radio', 'Radio'),
-    ('text', 'Text'),
-    ('text-file', 'Text File'),
-    ('toggle', 'Toggle'),
-]
-
-
-def validate_argument(argument_data):
-    error_dict = {}
-    key = validate_key(argument_data, error_dict)
-    if key is None:
-        return error_dict
-
-    error_dict[key] = {}
-    argument_error_dict = error_dict[key]
-
-    validate_unsupported_argument_fields(key, argument_data, argument_error_dict)
-
-    # Recursively validate sub_arguments
-    sub_arguments = argument_data.get('sub_arguments', {})
-    for name, sub_arguments in sub_arguments.items():
-        for subargument in sub_arguments:
-            errors = validate_argument(subargument)
-            if errors:
-                if 'sub_arguments' not in error_dict:
-                    argument_error_dict['sub_arguments'] = [errors]
-                else:
-                    argument_error_dict['sub_arguments'].append(validate_argument(subargument))
-
-    validate_required(key, argument_data, argument_error_dict)
-    type = validate_type(key, argument_data, argument_error_dict)
-
-    if not type:
-        return error_dict
-
-    validate_description(key, argument_data, type, argument_error_dict)
-
-    # Just return empty dict if we have no errors
-    if error_dict[key]:
-        return error_dict
-    else:
-        return {}
-
-
-def validate_key(argument_data, error_dict):
-    if 'key' not in argument_data.keys():
-        error_dict['required'] = [
-            f'One of your arguments is missing a key. Please specify a key for each of your arguments'
-        ]
-        return None
-    else:
-        return argument_data['key']
-
-
-def validate_required(key, argument_data, error_dict):
-    if 'required' in argument_data.keys():
-
-        if not isinstance(argument_data['required'], bool):
-            error_dict['required'] = [
-                f'Invalid value in required specified on {key} argument. required can be true or false'
-            ]
-
-
-def validate_type(key, argument_data, error_dict):
-    # Type is optional
-    if 'type' in argument_data.keys():
-
-        render_types_choices = [type_tuple[0] for type_tuple in render_types]
-        type = argument_data['type']
-        if type not in render_types_choices:
-            error_dict['type'] = [
-                f'Invalid value {type} in type specified on {key} argument \
-                type can be one of {render_types_choices}'
-            ]
-            return ''
-
-        if type == 'toggle':
-            if 'options' not in argument_data:
-                error_dict['type'] = [
-                    f'There must be exactly 2 options ("on" and "off") on arguments of type toggle'
-                ]
-                return ''
-
-            number_of_options = len(argument_data['options'].keys())
-
-            if number_of_options != 2:
-                error_dict['type'] = [
-                    f'There must be exactly 2 options ("on" and "off") on arguments of type toggle. Received \
-                    {number_of_options} options'
-                ]
-                return ''
-
-            option_names = list(argument_data['options'].keys())
-
-            if option_names not in (['on', 'off'], ['off', 'on']):
-                error_dict['type'] = [
-                    f'The two options on arguments of type toggle must be named "on" and "off". Received \
-                    {", ".join(option_names)}'
-                ]
-                return ''
-
-        return type
-
-
-def validate_description(key, argument_data, type, error_dict):
-    if 'description' not in argument_data.keys() and type != 'hidden':
-        error_dict['argument_description'] = [
-            f'Could not find a description for argument {key}. Please provide a description for {key}'
-        ]
-
-
-supported_argument_fields = [
-    'default_value',
-    'description',
-    'key',
-    'key_value_separator',
-    'options',
-    'required',
-    'sub_arguments',
-    'type',
-]
-
-
-def validate_unsupported_argument_fields(key, argument_data, error_dict):
-    for field in argument_data.keys():
-        if field not in supported_argument_fields:
-            error_dict['unsupported_field'] = [
-                f'The argument field {field} on {key} is not valid'
-            ]

biolib/validators/validate_module.py

@@ -1,323 +0,0 @@
-import re
-
-from biolib.validators.validator_utils import AllowedYAMLEnvironments, custom_executors, old_to_new_executors_map  # type: ignore
-
-
-def validate_module(name, module_data, yaml_version):
-    error_dict = {}
-    name = validate_name(name, error_dict)
-    if not name:
-        return error_dict
-    error_dict[name] = {}
-    task_error_dict = error_dict[name]
-
-    validate_unsupported_task_fields(name, module_data, task_error_dict, yaml_version)
-    validate_mappings(name, module_data, task_error_dict, mapping_type='input_files')
-    validate_mappings(name, module_data, task_error_dict, mapping_type='output_files')
-    validate_mappings(name, module_data, task_error_dict, mapping_type='source_files')
-    validate_image(name, module_data, task_error_dict, yaml_version)
-
-    validate_working_directory(name, module_data, task_error_dict)
-
-    # Just return empty dict if we have no errors
-    if error_dict[name]:
-        return error_dict
-    else:
-        return {}
-
-
-def validate_working_directory(name, task_data, error_dict):
-    if 'working_directory' in task_data:
-        if not task_data['working_directory'].startswith('/'):
-            error_dict['working_directory'] = [
-                f'Wrong path format on working_directory for {name}. Directory path must be an absolute path'
-            ]
-            return
-
-        if not task_data['working_directory'].endswith('/'):
-            error_dict['working_directory'] = [
-                f'Wrong path format on working_directory for {name}. Directories must end in a slash: "/dir/sub_dir/"'
-            ]
-            return
-
-        if '//' in task_data['working_directory']:
-            error_dict['working_directory'] = [
-                f'Wrong path format on working_directory for {name}. Directories can not have consecutive slashes"'
-            ]
-
-
-def validate_external_app_format(name, app_version_string, error_dict, yaml_version):
-    try:
-        split_on_slash = app_version_string.split('/')
-        if not len(split_on_slash) == 2:
-            if yaml_version == 1:
-                error_dict['external_app'] = [
-                    f'Wrong format. Tried to interpret {name} as an external-app; \
-                    The format should be owner/app:version i.e. "some_user/example_app:2.0.1.'
-                ]
-            else:
-                error_dict['image'] = [
-                    f'Wrong format. Tried to interpret {name} as a biolib app; \
-                    The format should be owner/app:version i.e. "some_user/example_app:2.0.1.'
-                ]
-            return None, None, None
-
-        account_handle, app_name_and_version = split_on_slash
-
-        split_on_colon = app_name_and_version.split(':')
-        if not len(split_on_slash) == 2:
-            if yaml_version == 1:
-                error_dict['task'] = [
-                    f'Wrong format. Tried to interpret {name} as an external-app; \
-                    The format should be owner/app:version i.e. "some_user/example_app:2.0.1.'
-                ]
-            else:
-                error_dict['image'] = [
-                    f'Wrong format. Tried to interpret {name} as a biolib app; \
-                    The format should be owner/app:version i.e. "some_user/example_app:2.0.1.'
-                ]
-            return None, None, None
-
-        app_name, version = split_on_colon
-        return account_handle, app_name, version
-
-    except:
-        if yaml_version == 1:
-            error_dict['external_app'] = [
-                f'Wrong format. Tried to interpret {name} as an external-app; \
-                The format should be owner/app:version i.e. "some_user/example_app:2.0.1.'
-            ]
-        else:
-            error_dict['image'] = [
-                f'Wrong format. Tried to interpret {name} as an external-app; \
-                The format should be owner/app:version i.e. "some_user/example_app:2.0.1.'
-            ]
-
-        return None, None, None
-
-
-def validate_name(name, error_dict):
-    # TODO: Refactor this to use a version of is_alphanumeric that allows control of the error message.
-    if not re.match("^[A-Za-z0-9_-]+$", name):
-        error_dict[name] = [f'The module name {name} is invalid, it can only contain alphanumeric characters.']
-        return
-
-    if re.search("(--)|(__)|(-_)|(_-)", name):
-        error_dict[name] = [f'The module name {name} is invalid, it can not contain consecutive dashes or underscores']
-        return
-
-    if re.match("^(-|_)[A-Za-z0-9_-]+$", name):
-        error_dict[name] = [f'The module name {name} is invalid, it can not start with dashes or underscores']
-        return
-
-    if re.match("^[A-Za-z0-9_-]+(-|_)$", name):
-        error_dict[name] = [f'The module name {name} is invalid, it can not end with dashes or underscores']
-        return
-
-    return name
-
-
-def validate_executor(name, task_data, error_dict):
-    if 'executor' not in task_data.keys():
-        error_dict['executor'] = [
-            'You must define an executor in your module definition; \
-            Make sure you follow the format executor_name:version'
-        ]
-        return
-
-    if task_data['executor'].count(':') != 1:
-        error_dict['executor'] = [
-            f'Executor {task_data["executor"]} on module {name} is invalid. Please only use ":" to separate to and from paths i.e. "from:to".'
-        ]
-        return
-
-    executor, version = task_data['executor'].split(':')
-    if executor not in old_to_new_executors_map.keys():
-        error_dict['executor'] = [
-            f'You provided an invalid executor in module {name}; \
-            Make sure you follow the format executor_name:version'
-        ]
-
-    new_executor_name = old_to_new_executors_map[executor]
-    supported_versions = custom_executors[new_executor_name]['versions'] + ['*']
-    if version not in supported_versions:
-        error_dict['image'] = [
-            f'Invalid version for executor {executor} on module {name}. The supported versions for {executor} are {supported_versions}'
-        ]
-        return
-
-
-def validate_mappings(name, task_data, error_dict, mapping_type):
-    """
-    Shared validation function for input and output mappings.
-    """
-    if mapping_type not in task_data:
-        error_dict[mapping_type] = [
-            f'{mapping_type} field on module {name} is required. Please specify your {mapping_type}.'
-        ]
-        return
-
-    if not isinstance(task_data[mapping_type], list):
-        error_dict[mapping_type] = [
-            f'{mapping_type} field on module {name} is invalid. Please format the field as a yaml array.'
-        ]
-        return
-
-    for mapping in task_data.get(mapping_type):
-        mapping_parts = mapping.split(' ')
-        if len(mapping_parts) != 3:
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} is invalid. Please use the format "COPY from_path to_path" i.e. "COPY / /home/biolib/"'
-            ]
-            return
-
-        if mapping_parts[0] != 'COPY':
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} is missing the COPY command. Please use the format "COPY from_path to_path" i.e. "COPY / /home/biolib/"'
-            ]
-            return
-
-        from_path = mapping_parts[1]
-        to_path = mapping_parts[2]
-
-        if '$' in re.sub(r"\$[1-9][0-9]*", "", from_path):
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} in path "{from_path}" is using an invalid variable. \
-Please only use variables referring to an argument number, where "$1" refers to the first argument \
-i.e. "COPY $1 /home/biolib/$1"'
-            ]
-            return
-
-        if '$' in re.sub(r"\$[1-9][0-9]*", "", to_path):
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} in path "{to_path}" is using an invalid variable. \
-Please only use variables referring to an argument number, where "$1" refers to the first argument \
-i.e. "COPY $1 /home/biolib/$1"'
-            ]
-            return
-
-        # Check that we don't map a directory to a file
-        to_path_with_vars_replaced_with_dollar = re.sub("\$[0-9]+", "$", to_path)
-        if from_path.endswith('/') and (not to_path.endswith('/') and
-                                        not to_path_with_vars_replaced_with_dollar.endswith('$')):
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} is invalid. Directories can only map to other directories'
-            ]
-            return
-
-        if not to_path.startswith('/') and not to_path.startswith('$'):
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} on path "{to_path}" is invalid. Only absolute paths allowed'
-            ]
-            return
-
-        if not from_path.startswith('/') and not from_path.startswith('$'):
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} on path "{from_path}" is invalid. Only absolute paths allowed'
-            ]
-            return
-
-        if '//' in from_path or '//' in to_path:
-            error_dict[mapping_type] = [
-                f'{mapping_type} item {mapping} on module {name} is invalid. Directories can not have consecutive slashes'
-            ]
-
-
-def validate_image(name, task_data, error_dict, yaml_version):
-    if 'image' not in task_data:
-        error_dict['image'] = [
-            f'You must define an image to use for module {name}.'
-        ]
-        return
-
-    image = task_data['image']
-    if '://' not in image:
-        error_dict['image'] = [
-            f'Wrong image format on module {name}. You must define an image using the following format "environment://image_name:version"'
-        ]
-        return
-
-    environment = image.split('://')[0]
-    if environment not in AllowedYAMLEnvironments.values():
-        error_dict['image'] = [
-            f'Wrong environment on image of module {name}. The environment should be specified before "://" and can be only be one of {AllowedYAMLEnvironments.values()}'
-        ]
-
-    if image.startswith(f'{AllowedYAMLEnvironments.BIOLIB_APP.value}://biolib/'):
-        # Image is a biolib custom executor
-        uri = image.replace(f'{AllowedYAMLEnvironments.BIOLIB_APP.value}://biolib/', '', 1)
-        if uri.count(':') != 1:
-            error_dict['image'] = [
-                f'Missing version on the image of module {name}. A version must be specified at the end of the image like so: "environment://image_name:version"'
-            ]
-            return
-
-        executor, version = uri.split(':')
-        # Check if the executor is supported
-        if executor not in custom_executors.keys():
-            error_dict['image'] = [
-                f'Invalid image name biolib/{executor} for biolib executor on module {name}. The supported biolib executors are {["biolib/" + executor for executor in custom_executors.keys()]}'
-            ]
-            return
-
-        # Check if the supplied version for the executor is supported
-        supported_versions = custom_executors[executor]['versions'] + ['*']
-        if version not in supported_versions:
-            error_dict['image'] = [
-                f'Invalid version for biolib executor {executor} on module {name}. The supported versions for {executor} are {supported_versions}'
-            ]
-            return
-
-    elif environment == AllowedYAMLEnvironments.BIOLIB_APP.value:
-        app_version_string = image.split('://')[1]
-        # validate_external_app(name, app_version_string, user, error_dict, yaml_version)
-
-    elif environment in (AllowedYAMLEnvironments.DOCKERHUB.value, AllowedYAMLEnvironments.LOCAL_DOCKER.value):
-        repo_and_tag = image.split('://')[1]
-        if not repo_and_tag.count(':', 1):
-            error_dict['image'] = [
-                f'Invalid docker image on module {name}. A tag must be included in your image with format repo:tag i.e. alpine:latest'
-            ]
-            return
-
-
-# Task fields shared between all versions
-supported_task_fields_base = [
-    'working_directory',
-]
-
-supported_task_fields_v1 = [
-    'executor',
-    'path'
-]
-
-supported_task_fields_v2 = [
-    'image',
-    'input_files',
-    'output_files',
-    'source_files',
-    'command',
-]
-
-
-def validate_unsupported_task_fields(name, task_data, error_dict, yaml_version):
-    # If we need to validate module fields, i.e. the module is not a string in v1, then it has to be a dict
-    if not isinstance(task_data, dict):
-        error_dict['unsupported_fields'] = [
-            f'Module {name} is the wrong type. Modules can only be a YAML dict in version {yaml_version}']
-        return
-
-    if yaml_version == 1:
-        supported_fields = supported_task_fields_base + supported_task_fields_v1
-    else:
-        supported_fields = supported_task_fields_base + supported_task_fields_v2
-
-    errors = []
-    for field in task_data.keys():
-        if field not in supported_fields:
-            errors.append(
-                f'The module field {field} on {name} is not valid for biolib yaml version {yaml_version}'
-            )
-
-    if errors:
-        error_dict['unsupported_fields'] = errors

biolib/validators/validate_zip_file.py

@@ -1,40 +0,0 @@
-from zipfile import ZipFile
-
-
-def validate_zip_file(zip_file_path, app_path):
-    try:
-        zip_file = ZipFile(zip_file_path)
-    except Exception:
-        raise Exception('Failed to read source files zip for validation.') from None
-
-    files = [filename.strip('/') for filename in zip_file.namelist()]
-    top_level_files = [filename for filename in files if len(filename.split('/')) == 1]
-
-    if len(top_level_files) > 1:
-        raise Exception('Please place all your source files in a single folder and zip it')
-
-    # Check if .biolib folder is present
-    if f'{top_level_files[0]}/.biolib' in files and f'{top_level_files[0]}/biolib' in files:
-        raise Exception('You provided both a biolib and a .biolib folder. Please only provide the .biolib folder')
-
-    elif f'{top_level_files[0]}/.biolib' in files:
-        biolib_folder_path = f'{top_level_files[0]}/.biolib'
-
-    elif f'{top_level_files[0]}/biolib' in files:
-        raise Exception('Your biolib folder appears to be called "biolib" - it must be called ".biolib".')
-
-    else:
-        raise Exception(f'Could not find a .biolib folder in provided application folder {app_path}')
-
-    # Check if the config file is present
-    if f'{biolib_folder_path}/config.yml' in files:
-        # TODO: Changed to make sense in CLI. Rewrite this check at some point
-        pass
-
-    elif f'{biolib_folder_path}/config.yaml' in files:
-        raise Exception('Your biolib config file has the .yaml file extension - it must be .yml')
-
-    else:
-        raise Exception(
-            f'Could not find config.yml file. Please provide a yaml file named config.yml in the .biolib folder'
-        )