conviso-ast 3.0.0__py3-none-any.whl

convisoappsec/flowcli/vulnerability/run.py

@@ -0,0 +1,487 @@
+import click
+import json
+from convisoappsec.flowcli import help_option
+from convisoappsec.flowcli.context import pass_flow_context
+from convisoappsec.sast.sastbox import SASTBox
+from convisoappsec.flowcli.common import (asset_id_option, on_http_error)
+from convisoappsec.flowcli.requirements_verifier import RequirementsVerifier
+from convisoappsec.flow import GitAdapter
+from convisoappsec.common.box import ContainerWrapper
+from convisoappsec.common.graphql.errors import ResponseError
+from convisoappsec.flow.cleaner import RunnerCleanup
+from convisoappsec.logger import LOGGER
+
+
+@click.command()
+@asset_id_option(required=False)
+@click.option(
+    "-s",
+    "--start-commit",
+    required=False,
+    help="If no value is set so the empty tree hash commit is used.",
+)
+@click.option(
+    "-e",
+    "--end-commit",
+    required=False,
+    help="""If no value is set so the HEAD commit
+    from the current branch is used""",
+)
+@click.option(
+    "-r",
+    "--repository-dir",
+    default=".",
+    show_default=True,
+    type=click.Path(
+        exists=True,
+        resolve_path=True,
+    ),
+    required=False,
+    help="The source code repository directory.",
+)
+@click.option(
+    "--fail-on-severity-threshold",
+    required=False,
+    help="If the threshold of the informed severity and higher has reach, then the command will fail after send the results to AppSec Flow.\n \
+    The severity levels are: UNDEFINED, INFO, LOW, MEDIUM, HIGH, CRITICAL.",
+    type=click.Tuple([str, int]),
+    default=(None, None),
+)
+@click.option(
+    "--fail-on-threshold",
+    required=False,
+    help="If the threshold has reach then the command will fail after send the result to AppSec Flow",
+    type=int,
+    default=False,
+)
+@click.option(
+    "--send-to-flow/--no-send-to-flow",
+    default=True,
+    show_default=True,
+    required=False,
+    hidden=True,
+    help="""Enable or disable the ability of send analysis result
+    reports to flow.""",
+)
+@click.option(
+    "--deploy-id",
+    default=None,
+    required=False,
+    hidden=True,
+    envvar=("CONVISO_DEPLOY_ID", "FLOW_DEPLOY_ID")
+)
+@click.option(
+    "--sastbox-registry",
+    default="",
+    required=False,
+    hidden=True,
+    envvar=("CONVISO_SASTBOX_REGISTRY", "FLOW_SASTBOX_REGISTRY"),
+)
+@click.option(
+    "--sastbox-repository-name",
+    default="",
+    required=False,
+    hidden=True,
+    envvar=("CONVISO_SASTBOX_REPOSITORY_NAME", "FLOW_SASTBOX_REPOSITORY_NAME"),
+)
+@click.option(
+    "--sastbox-tag",
+    default=SASTBox.DEFAULT_TAG,
+    required=False,
+    hidden=True,
+    envvar=("CONVISO_SASTBOX_TAG", "FLOW_SASTBOX_TAG"),
+)
+@click.option(
+    "--sastbox-skip-login/--sastbox-no-skip-login",
+    default=False,
+    required=False,
+    hidden=True,
+    envvar=("CONVISO_SASTBOX_SKIP_LOGIN", "FLOW_SASTBOX_SKIP_LOGIN"),
+)
+@click.option(
+    '--experimental',
+    default=False,
+    is_flag=True,
+    hidden=True,
+    help="Enable experimental features.",
+)
+@click.option(
+    "--company-id",
+    required=False,
+    envvar=("CONVISO_COMPANY_ID", "FLOW_COMPANY_ID"),
+    help="Company ID on Conviso Platform",
+)
+@click.option(
+    '--asset-name',
+    required=False,
+    envvar=("CONVISO_ASSET_NAME", "FLOW_ASSET_NAME"),
+    help="Provides a asset name.",
+)
+@click.option(
+    '--vulnerability-auto-close',
+    default=False,
+    is_flag=True,
+    hidden=True,
+    help="Enable auto fixing vulnerabilities on cp.",
+)
+@click.option(
+    '--from-ast',
+    default=False,
+    is_flag=True,
+    hidden=True,
+    help="Internal use only.",
+)
+@click.option(
+    '--cleanup',
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Clean up system resources, including temporary files, stopped containers, unused Docker images and volumes.",
+)
+@click.option(
+    '--control-sync-status-id',
+    required=False,
+    hidden=True,
+    help="Control sync status id.",
+)
+@help_option
+@pass_flow_context
+@click.pass_context
+def run(context, flow_context, asset_id, company_id, end_commit, start_commit, repository_dir, send_to_flow, deploy_id,
+        sastbox_registry, sastbox_repository_name, sastbox_tag, sastbox_skip_login, fail_on_threshold,
+        fail_on_severity_threshold, experimental, asset_name, vulnerability_auto_close, from_ast, cleanup,
+        control_sync_status_id):
+
+    if context.params.get('cleanup'):
+        try:
+            log_func("Cleaning before runs auto close ...")
+            cleaner = RunnerCleanup()
+            cleaner.cleanup_all()
+        except Exception as e:
+            log_func(f"Failed to clean, trying any way ... Error: {str(e)}")
+
+    if not from_ast:
+        prepared_context = RequirementsVerifier.prepare_context(context)
+
+        params_to_copy = [
+            'asset_id', 'company_id', 'start_commit', 'end_commit',
+            'repository_dir', 'send_to_flow', 'deploy_id', 'sastbox_registry',
+            'sastbox_repository_name', 'sastbox_tag', 'sastbox_skip_login',
+            'experimental', 'asset_name', 'vulnerability_auto_close', 'company_id'
+        ]
+
+        for param_name in params_to_copy:
+            context.params[param_name] = (
+                    locals()[param_name] or prepared_context.params[param_name]
+            )
+
+    conviso_api = flow_context.create_conviso_api_client_beta()
+    company_id = company_id or context.params['company_id']
+    asset_id = context.params['asset_id']
+    statuses = ['CREATED', 'IDENTIFIED', 'IN_PROGRESS', 'AWAITING_VALIDATION', 'FIX_ACCEPTED']
+
+    page = 1
+    merged_issues_sast = []
+    merged_issues_sca = []
+
+    log_func("Running automatic closure of vulnerabilities...")
+
+    while True:
+        try:
+            issues_from_cp = conviso_api.issues.auto_close_vulnerabilities(
+                company_id, asset_id, statuses, page
+            )
+        except ResponseError as error:
+            if 'Variable $company_id' in str(error):
+                log_func(f"Invalid company_id passed: {company_id}", fg='red')
+            else:
+                log_func(f"error: {error}", fg='red')
+
+            log_func("⚠️ Auto-close will not be performed at this time. Please set it using --company-id and try again.", fg='red')
+            return
+
+        total_pages = issues_from_cp['metadata']['totalPages']
+        issues_collection = issues_from_cp['collection']
+        issues_collection = [item for item in issues_collection if item['scanSource'] == 'conviso_ast']
+        sast_issues = [item for item in issues_collection if item['type'] == 'SAST_FINDING']
+        sca_issues = [item for item in issues_collection if item['type'] == 'SCA_FINDING']
+
+        # could not be if and else because on the response could contain both.
+        if sast_issues:
+            merged_issues_sast.extend(sast_issues)
+
+        if sca_issues:
+            merged_issues_sca.extend(sca_issues)
+
+        if total_pages == 0 or total_pages == page:
+            break
+        else:
+            page += 1
+
+    # sast issues filter
+    sast_issues_with_fix_accepted = [item for item in merged_issues_sast if item['status'] == 'FIX_ACCEPTED']
+    sast_issues_without_fix_accepted = [item for item in merged_issues_sast if item['status'] != 'FIX_ACCEPTED']
+
+    # sca issues filter
+    sca_issues_with_fix_accepted = [item for item in merged_issues_sca if item['status'] == 'FIX_ACCEPTED']
+    sca_issues_without_fix_accepted = [item for item in merged_issues_sca if item['status'] != 'FIX_ACCEPTED']
+
+    if len(issues_from_cp) == 0:
+        log_func("No vulnerabilities were found on the Conviso Platform!")
+        return
+
+    # Starting executing the ast again
+    sast_hash_issues = perform_sastbox_scan(sastbox_registry, sastbox_repository_name, sastbox_tag, repository_dir)
+
+    sca_hash_issues = perform_sca_scan(repository_dir=repository_dir)
+    iac_hash_issues = perform_iac_scan(repository_dir=repository_dir)
+
+    # we need to append the two lists because at the moment this was made, iac and sast has sast as type on cp.
+    sast_hash_issues = sast_hash_issues + iac_hash_issues
+    # end ast execution
+
+    set_of_sast_hash_issues = set(sast_hash_issues)
+    set_of_sca_hash_issues = set(sca_hash_issues)
+
+    close_sast_issues(conviso_api, sast_issues_without_fix_accepted, set_of_sast_hash_issues, control_sync_status_id)
+    close_sca_issues(conviso_api, sca_issues_without_fix_accepted, set_of_sca_hash_issues, control_sync_status_id)
+
+    sast_issues_to_reopen = [
+        {'id': item['id'], 'originalIssueIdFromTool': item['originalIssueIdFromTool']}
+        for item in sast_issues_with_fix_accepted if item['originalIssueIdFromTool'] in sast_hash_issues
+    ]
+
+    sca_issues_to_reopen = [
+        {'id': item['id'], 'originalIssueIdFromTool': item['originalIssueIdFromTool']}
+        for item in sca_issues_with_fix_accepted if item['originalIssueIdFromTool'] in sca_hash_issues
+    ]
+
+    if sast_issues_to_reopen:
+        log_func("SAST: reopening {issues} vulnerability/vulnerabilities on conviso platform ...".format(
+            issues=len(sast_issues_to_reopen))
+        )
+
+        reopen_issues(conviso_api, sast_issues_to_reopen, control_sync_status_id)
+
+    if sca_issues_to_reopen:
+        log_func("SCA: reopening {issues} vulnerability/vulnerabilities on conviso platform ...".format(
+            issues=len(sca_issues_to_reopen))
+        )
+
+        reopen_issues(conviso_api, sca_issues_to_reopen, control_sync_status_id)
+
+
+def close_sast_issues(conviso_api, issues_from_cp, issues_from_current_scan, control_sync_status_id):
+    """
+    method to close sast issues on conviso platform
+
+    # issues_from_cp are issues already on conviso platform.
+    # issues_from_current_scan are issues identify on each time the ast command runs, these are always from a full code
+    base scan
+    """
+    log_func("SAST: Verifying if any vulnerability was solved...")
+
+    differences = [
+        {'id': item['id'], 'originalIssueIdFromTool': item['originalIssueIdFromTool']}
+        for item in issues_from_cp if item['originalIssueIdFromTool'] not in issues_from_current_scan
+    ]
+
+    if len(differences) == 0:
+        log_func("No vulnerabilities have been fixed yet...")
+        return
+
+    log_func("SAST: Fixing {issues} vulnerabilities on the Conviso Platform...".format(issues=len(differences)))
+
+    for issue in differences:
+        issue_id = issue['id']
+        status = 'FIX_ACCEPTED'
+        reason = ("The vulnerability is no longer found in the specified file/path. Its status has been updated by "
+                  "Conviso AST")
+
+        conviso_api.issues.update_issue_status(
+            issue_id=issue_id, status=status, reason=reason, control_sync_status_id=control_sync_status_id
+        )
+
+
+def close_sca_issues(conviso_api, issues_from_cp, issues_from_current_scan, control_sync_status_id):
+    """ method to close sca issues on conviso platform """
+
+    log_func("SCA: Verifying if any vulnerability was solved...")
+    differences = [
+        {'id': item['id'], 'originalIssueIdFromTool': item['originalIssueIdFromTool']}
+        for item in issues_from_cp if item['originalIssueIdFromTool'] not in issues_from_current_scan
+    ]
+
+    if len(differences) == 0:
+        log_func("No vulnerabilities have been fixed yet...")
+        return
+
+    log_func("SCA: Fixing {issues} vulnerabilities on the Conviso Platform...".format(issues=len(differences)))
+
+    for issue in differences:
+        issue_id = issue['id']
+        status = 'FIX_ACCEPTED'
+        reason = ("The vulnerability is no longer found in the specified file/path. Its status has been updated by "
+                  "Conviso AST")
+
+        conviso_api.issues.update_issue_status(
+            issue_id=issue_id, status=status, reason=reason, control_sync_status_id=control_sync_status_id
+        )
+
+
+def reopen_issues(conviso_api, issues, control_sync_status_id):
+    """ Method to reopen issues when was detected again after a full scan is performed """
+
+    for issue in issues:
+        issue_id = issue['id']
+        status = 'IDENTIFIED'
+        reason = 'Status has been updated from Fixed to Identified by Conviso AST'
+
+        conviso_api.issues.update_issue_status(
+            issue_id=issue_id, status=status, reason=reason, control_sync_status_id=control_sync_status_id
+        )
+
+def perform_sastbox_scan(sastbox_registry, sastbox_repository_name, sastbox_tag, repository_dir):
+    LOGGER.info('   🔍 [SAST] Running scan...')
+
+    sastbox = SASTBox(registry=sastbox_registry, repository_name=sastbox_repository_name, tag=sastbox_tag)
+    git_adapter = GitAdapter(repository_dir, unshallow_repository=False)
+
+    reports = sastbox.run_scan_diff(
+        repository_dir,
+        git_adapter.head_commit,
+        git_adapter.empty_repository_tree_commit
+    )
+
+    try:
+        first_report = next(reports)
+        with open(str(first_report), 'r') as f:
+            issues = json.load(f).get("issues", [])
+
+        return [
+            issue.get("hash_issue") or issue.get("hash_issue_v2")
+            for issue in issues
+            if issue.get("hash_issue") or issue.get("hash_issue_v2")
+        ]
+
+    except (StopIteration, Exception) as e:
+        if isinstance(e, StopIteration):
+            return []
+        return []
+
+@pass_flow_context
+def perform_sca_scan(flow_context, repository_dir):
+    LOGGER.info('   🔍 [SCA] Running scan...')
+
+    try:
+        REQUIRED_CODEBASE_PATH = '/code'
+        OSV_SCANNER_IMAGE_NAME = 'osv_scanner'
+
+        scanners = {
+            OSV_SCANNER_IMAGE_NAME: {
+                'repository_name': OSV_SCANNER_IMAGE_NAME,
+                'tag': 'latest',
+                'command': [
+                    '-c', REQUIRED_CODEBASE_PATH,
+                    '-f', 'json',
+                    '-o', '/{}.json'.format(OSV_SCANNER_IMAGE_NAME)
+                ],
+                'repository_dir': repository_dir
+            },
+        }
+
+        conviso_rest_api = flow_context.create_conviso_rest_api_client()
+        token = conviso_rest_api.docker_registry.get_sast_token()
+        scabox = ContainerWrapper(
+            token=token,
+            containers_map=scanners,
+            logger=None,
+            timeout=7200
+        )
+
+        scabox.run()
+
+        results_filepaths = []
+        for unit in scabox.scanners:
+            file_path = unit.results
+            if file_path:
+                results_filepaths.append(file_path)
+
+        hash_issues = []
+
+        for report_path in results_filepaths:
+            try:
+                with open(report_path, 'r') as report_file:
+                    report_content = json.load(report_file)
+                    issues = report_content.get("issues", [])
+                    hash_issues.extend(issue.get("hash_issue") for issue in issues)
+
+            except (FileNotFoundError, json.JSONDecodeError) as e:
+                print(f"Error processing {report_path}: {e}")
+
+        return hash_issues
+
+    except Exception as e:
+        on_http_error(e)
+        raise click.ClickException(str(e)) from e
+
+
+@pass_flow_context
+def perform_iac_scan(flow_context, repository_dir):
+    """ Perform an iac scan """
+    LOGGER.info('   🔍 [IaC] Running scan...')
+
+    try:
+        REQUIRED_CODEBASE_PATH = '/code'
+        IAC_IMAGE_NAME = 'iac_scanner_checkov'
+        IAC_SCAN_FILENAME = '/{}.json'.format(IAC_IMAGE_NAME)
+        containers_map = {
+            IAC_IMAGE_NAME: {
+                'repository_dir': repository_dir,
+                'repository_name': IAC_IMAGE_NAME,
+                'tag': 'unstable',
+                'command': [
+                    '-c', REQUIRED_CODEBASE_PATH,
+                    '-o', IAC_SCAN_FILENAME,
+                ],
+            },
+        }
+
+        conviso_rest_api = flow_context.create_conviso_rest_api_client()
+        token = conviso_rest_api.docker_registry.get_sast_token()
+        scanners_wrapper = ContainerWrapper(
+            token=token,
+            containers_map=containers_map,
+            logger=None,
+            timeout=7200
+        )
+
+        scanners_wrapper.run()
+
+        results_filepaths = []  # [str(r.results) for r in scanners_wrapper.scanners]
+        hash_issues = []
+        for r in scanners_wrapper.scanners:
+            report_filepath = r.results
+            if report_filepath:
+                results_filepaths.append(report_filepath)
+
+        for report_path in results_filepaths:
+            try:
+                with open(report_path) as report_file:
+                    report_content = json.load(report_file)
+                    results = report_content.get("runs", [])[0].get("results", [])
+                    for result in results:
+                        partial_fingerprints = result.get("partialFingerprints", {})
+                        hash_issue = partial_fingerprints.get("hashIssueV2")
+                        if hash_issue:
+                            hash_issues.append(hash_issue)
+            except (FileNotFoundError, json.JSONDecodeError) as e:
+                print(f"Error processing {report_path}: {e}")
+
+        return hash_issues
+    except Exception as e:
+        on_http_error(e)
+        raise click.ClickException(str(e)) from e
+
+def log_func(msg, new_line=True, fg='blue'):
+    click.echo(click.style(msg, bold=True, fg=fg), nl=new_line, err=True)

convisoappsec/logger.py

@@ -0,0 +1,29 @@
+import logging
+
+logging.basicConfig(
+    filename='output.log',
+    filemode='w',
+    level=logging.DEBUG,
+    format='%(levelname)s:  %(filename)s:  %(threadName)s:  %(name)s:  %(message)s'
+)
+
+LOGGER = logging.getLogger(__name__)
+
+
+def log_and_notify_ast_event(flow_context, company_id, asset_id, ast_log):
+    """
+    Logs an AST (Application Security Test) event for a specific company and asset,
+    and sends notifications.
+
+    This method performs the following actions:
+    1. Sends the provided AST log to a monitoring platform like Datadog for logging and tracking.
+    2. Sends a Slack notification with relevant details about the AST event.
+
+    Args:
+        flow_context: The context of the AST execution.
+        company_id (int): The ID of the company for which the AST event is being logged.
+        asset_id (int): The ID of the asset associated with the AST event.
+        ast_log (string): A string with the log details of the AST event.
+    """
+    conviso_api = flow_context.create_conviso_graphql_client()
+    conviso_api.ast_errors.send_ast_error(company_id = company_id, asset_id = asset_id, log = ast_log)

convisoappsec/sast/decision.py

@@ -0,0 +1,45 @@
+from enum import IntEnum
+import json
+
+class Severity(IntEnum):
+    # Sastbox [:undefined, :info, :low, :medium, :high, :critical]
+    UNDEFINED = 0
+    INFO = 1
+    LOW = 2
+    MEDIUM = 3
+    HIGH = 4
+    CRITICAL = 5
+    
+    @classmethod
+    def has_value(cls, value):
+        return value in cls._member_names_
+
+
+
+class Decision:
+
+
+    def __init__(self, report):
+        self.report = open(report)
+        self.json = json.load(self.report)
+        self.issues = self.json.get('issues')
+        self.sev = self.issues
+
+    def block_from_severity(self, severity, threshold=1):
+        if not severity:
+            return False
+        severity = Severity[severity]
+        self.sev = [issue for issue in self.issues if Severity[issue.get('severity').upper()] >= severity]
+        return len(self.sev) >= threshold
+
+    def block_from_findings(self, threshold):
+        if not threshold:
+            return False
+        return len(self.issues) >= threshold
+    
+    def filtered_issues(self):
+        return self.sev
+
+
+
+