regscale-cli 6.16.0.0__py3-none-any.whl

regscale/integrations/commercial/gitlab.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""RegScale Gitlab integration"""
+
+# standard python imports
+import os
+import re
+import sys
+from urllib.parse import urljoin
+
+import click
+import markdown
+import requests
+from rich.progress import Progress
+
+from regscale.core.app.internal.login import is_valid
+from regscale.core.app.logz import create_logger
+from regscale.core.app.utils.app_utils import (
+    check_license,
+    create_progress_object,
+    error_and_exit,
+    get_current_datetime,
+)
+from regscale.models import regscale_id, regscale_module
+from regscale.models.regscale_models.issue import Issue
+from regscale.models.regscale_models.link import Link
+
+job_progress = create_progress_object()
+logger = create_logger()
+
+
+@click.group()
+def gitlab():
+    """GitLab integration to pull issues via API."""
+
+
+@gitlab.command(name="sync_issues", help="Integration to GitLab to sync issues into a module.")
+@regscale_id()
+@regscale_module()
+@click.option("--gitlab_url", "-u", default="https://gitlab.com", help="GitLab URL", required=True)
+@click.option(
+    "--gitlab_project_id",
+    "-gpid",
+    required=True,
+    help="The ID of the GitLab project to pull issues from.",
+    default=os.environ.get("GITLAB_PROJECT"),
+)
+@click.option(
+    "--api_token",
+    "-t",
+    required=True,
+    help="Your GitLab API token with API read access.",
+    default=os.environ.get("GITLAB_API_TOKEN"),
+)
+@click.option(
+    "--include_links",
+    "-l",
+    is_flag=True,
+    help="Include links from the issue description.",
+    default=False,
+)
+def sync_issues(
+    regscale_id: int,
+    regscale_module: str,
+    gitlab_url: str,
+    gitlab_project_id: int,
+    api_token: str,
+    include_links: bool,
+):
+    """Sync issues from a GitLab project into a RegScale record."""
+    run_sync_issues(
+        regscale_id=regscale_id,
+        regscale_module=regscale_module,
+        gitlab_url=gitlab_url,
+        gitlab_project_id=gitlab_project_id,
+        api_token=api_token,
+        include_links=include_links,
+    )
+
+
+def run_sync_issues(
+    regscale_id: int,
+    regscale_module: str,
+    gitlab_url: str,
+    gitlab_project_id: int,
+    api_token: str,
+    include_links: bool,
+) -> None:
+    """Sync issues from a GitLab project into a module
+
+    :param int regscale_id: The RegScale ID to sync issues to
+    :param str regscale_module: The RegScale module to sync issues to
+    :param str gitlab_url: The GitLab URL to sync issues from
+    :param int gitlab_project_id: The GitLab project ID to sync issues from
+    :param str api_token: The GitLab API token to use
+    :param bool include_links: Whether to include links from the issue description
+    :rtype: None
+    """
+    app = check_license()
+    if not is_valid(app=app):
+        logger.warn("RegScale token is invalid. please login.")
+        sys.exit(1)
+
+    with job_progress:
+        gitlab_issues = get_issues_from_gitlab(gitlab_url, gitlab_project_id, api_token, job_progress)
+
+        regscale_issues = get_regscale_issues(regscale_id, regscale_module, job_progress)
+        logger.debug(f"Fetched {len(regscale_issues)} issues from RegScale.")
+
+        # Convert the issues to your desired format
+        issues = convert_issues(
+            gitlab_issues,
+            regscale_id,
+            regscale_module,
+            include_links,
+            job_progress,
+        )
+
+        # # Save or update the converted issues
+        save_or_update_issues(issues, regscale_issues, job_progress)
+
+
+def get_regscale_issues(regscale_id: int, regscale_module: str, job_progress: Progress) -> list:
+    """Function to fetch issues from RegScale
+
+    :param int regscale_id: The RegScale ID to fetch issues for
+    :param str regscale_module: The RegScale module to fetch issues for
+    :param Progress job_progress: The progress object to use for updating
+    :return: list of regscale issues
+    :rtype: list
+    """
+    app = check_license()
+    task = job_progress.add_task("[#f8b737]Fetching issues from regscale", total=1)
+    if regscale_module == "securityplans":
+        existing_issues = Issue.fetch_issues_by_ssp(app=app, ssp_id=regscale_id)
+        logger.info(f"Fetched {len(existing_issues)} issues from RegScale by SSP.")
+    else:
+        existing_issues = Issue.fetch_issues_by_parent(
+            app=app, regscale_id=regscale_id, regscale_module=regscale_module
+        )
+        logger.info(f"Fetched {len(existing_issues)} issues from RegScale by issue parent.")
+    job_progress.update(task, advance=1)
+    return existing_issues
+
+
+def save_or_update_issues(gitlab_issues: list, regscale_issues: list, job_progress: Progress) -> None:
+    """Function to save or update issues from GitLab to RegScale
+
+    :param list gitlab_issues: The list of GitLab issues to save or update
+    :param list regscale_issues: The list of RegScale issues to save or update
+    :param Progress job_progress: The progress object to use for updating
+    :rtype: None
+    """
+    app = check_license()
+    # figure out which issues need to be updated vs inserted
+    task = job_progress.add_task(
+        "[#f8b737]Saving issues from GitLab to RegScale...",
+        total=len(gitlab_issues),
+    )
+    regscale_dict = {regscale_issue.dependabotId: regscale_issue for regscale_issue in regscale_issues}
+
+    for gitlab_issue_obj in gitlab_issues:
+        gitlab_issue = gitlab_issue_obj.get("issue")
+        # if we have the issue already in the regscale dict, check and update it
+        if gitlab_issue.dependabotId in regscale_dict:
+            regscale_issue = regscale_dict.get(gitlab_issue.dependabotId)
+            # has it updated?
+            if regscale_issue.__eq__(gitlab_issue) is False:
+                gitlab_issue.id = regscale_issue.id
+                try:
+                    Issue.update_issue(app=app, issue=gitlab_issue)
+                    logger.info(f"Updated issue {gitlab_issue.id}")
+                except Exception as vex:
+                    logger.error(vex)
+                existing_links = Link.fetch_links_by_parent(app, gitlab_issue.id, "issues")
+                for link in gitlab_issue_obj.get("links", []):
+                    # set here for comparison purposes
+                    link.parentID = gitlab_issue.id
+                    if link not in existing_links:
+                        try:
+                            new_link = Link.insert_link(app=app, link=link)
+                            logger.info(f"Inserted link {new_link.id}")
+                            # Add the new link to the existing_links list
+                            existing_links.append(new_link)
+                        except Exception as ex:
+                            logger.error(ex)
+
+        # insert new issue
+        else:
+            try:
+                issue = Issue.insert_issue(app=app, issue=gitlab_issue)
+                if issue is not None and issue.id is not None:
+                    logger.info(f"Inserted issue {issue.id}")
+            except Exception as ex:
+                logger.error(ex)
+            for link in gitlab_issue_obj.get("links", []):
+                link.parentID = issue.id
+                try:
+                    new_link = Link.insert_link(app=app, link=link)
+                except Exception as ex:
+                    logger.error(ex)
+        job_progress.update(task, advance=1)
+
+
+def extract_links_with_labels(text: str, parent_id: int, parent_module: str) -> list[Link]:
+    """Extract links from an issue description text with labels
+
+    :param str text: The issue description containing links
+    :param int parent_id: The parent ID associated with the parent module
+    :param str parent_module: The parent module associated with the parent ID
+    :return: A list of Link objects extracted from the text
+    :rtype: list[Link]
+    """
+    results = []
+    url_pattern = re.compile(r"https?://(?:[a-zA-Z0-9$@.&+*(),%-]|%[0-9a-fA-F]{2})+")
+
+    for line in text.split("\n"):
+        if ":" in line and ("Link" in line or "link" in line):
+            label, url = line.split(":", 1)
+            url = url.strip().replace("<br>", "")
+
+            if url.startswith("https:"):
+                url = url[6:].strip()
+
+            if url_pattern.match(url):
+                results.append(
+                    Link(
+                        title=label.replace("-", "").strip(),
+                        url=url,
+                        parentID=parent_id,
+                        parentModule=parent_module,
+                    )
+                )
+
+    return results
+
+
+def convert_issues(
+    gitlab_issues: list,
+    regscale_id: int,
+    regscale_module: str,
+    include_links: bool,
+    job_progress: Progress,
+) -> list:
+    """
+    Converts issues from GitLab to regscale
+
+    :param list gitlab_issues: The list of GitLab issues to convert
+    :param int regscale_id: The RegScale ID to convert issues to
+    :param str regscale_module: The RegScale module to convert issues to
+    :param bool include_links: Whether to include links from the issue description
+    :param Progress job_progress: The progress object to use for updating
+    :return: list of converted issues
+    :rtype: list
+    """
+    app = check_license()
+
+    task = job_progress.add_task("[#f8b737]Converting issues from gitlab...", total=len(gitlab_issues))
+    regscale_issues = []
+    for issue in gitlab_issues:
+        status = "Open"
+        if issue.get("state"):
+            if issue.get("state") == "open":
+                status = "Open"
+            elif issue.get("state") == "closed":
+                status = "Closed"
+        severity_level = Issue.assign_severity(issue.get("weight", 0))
+        # Convert the issue to your desired format
+        converted_issue = Issue(
+            title=issue["title"],
+            description=str(markdown.markdown(issue["description"])),
+            severityLevel=severity_level,
+            issueOwnerId=app.config["userId"],
+            costEstimate=0,
+            levelOfEffort=0,
+            dueDate=issue["due_date"],
+            identification="Other",
+            dependabotId=str(issue["id"]),
+            dateCreated=issue["created_at"],
+            parentId=regscale_id,
+            parentModule=regscale_module,
+            status=status,
+            securityPlanId=regscale_id if regscale_module == "securityplans" else None,
+            componentId=regscale_id if regscale_module == "components" else None,
+        )
+        if converted_issue.status == "Closed":
+            if issue.get("closed_at"):
+                converted_issue.dateCompleted = issue.get("closed_at")
+            else:
+                converted_issue.dateCompleted = get_current_datetime()
+        # Extract the links from the description
+        if include_links:
+            links = extract_links_with_labels(issue["description"], 0, "issues")
+            regscale_issues.append({"issue": converted_issue, "links": links})
+        else:
+            regscale_issues.append({"issue": converted_issue, "links": []})
+        job_progress.update(task, advance=1)
+    return regscale_issues
+
+
+def get_issues_from_gitlab(gitlab_url: str, gitlab_project_id: int, api_token: str, job_progress: Progress) -> list:
+    """Fetch issues from GitLab
+
+    :param str gitlab_url: The GitLab URL to fetch issues from
+    :param int gitlab_project_id: The GitLab project ID to fetch issues from
+    :param str api_token: The GitLab API token to use
+    :param Progress job_progress: The progress object to use for updating
+    :return: list of issues
+    :rtype: list
+    """
+    # Define the GitLab API URL for issues
+    api_call = f"/api/v4/projects/{gitlab_project_id}/issues"
+    url = urljoin(gitlab_url, api_call)
+    logger.info("Fetching issues from gitlab...")
+    logger.debug(f"Fetching with API token {api_token}")
+    # Define the headers, including your API token
+    headers = {"Private-Token": api_token}
+    # Send a GET request to the API
+    fetching_issues = job_progress.add_task("[#f8b737]Fetching issues from gitlab...", total=1)
+    response = requests.get(url, headers=headers)
+    job_progress.update(fetching_issues, advance=1)
+    issues = []
+    # If the request was successful
+    if response.ok:
+        # Load the issues from the response
+        issues = response.json()
+        logger.info(f"Fetched {len(issues)} issues from gitlab")
+    else:
+        logger.error(response.status_code)
+        logger.error(response.text)
+        error_and_exit(f"Failed to get issues from GitLab. Status code: {response.status_code}")
+    return issues

regscale/integrations/commercial/grype.py

@@ -0,0 +1,165 @@
+"""
+Module for processing Grype scan results and loading them into RegScale.
+"""
+
+import logging
+import traceback
+from datetime import datetime
+from typing import List, Optional, Union
+
+import click
+from pathlib import Path
+
+from regscale.core.app.utils.file_utils import (
+    download_from_s3,
+    find_files,
+    iterate_files,
+    move_file,
+)
+from regscale.models.integration_models.flat_file_importer import FlatFileImporter
+from regscale.models.integration_models.grype_import import GrypeImport
+
+logger = logging.getLogger(__name__)
+
+
+class GrypeProcessingError(Exception):
+    """Custom exception for Grype processing errors."""
+
+    pass
+
+
+@click.group()
+def grype():
+    """Performs actions from the Grype scanner integration."""
+    pass
+
+
+@grype.command("import_scans")
+@FlatFileImporter.common_scanner_options(
+    message="File path to the folder containing Grype .json files to process to RegScale.",
+    prompt="File path for Grype files",
+    import_name="grype",
+)
+@click.option("--destination", "-d", type=click.Path(exists=True, dir_okay=True), required=False)
+@click.option("--file_pattern", "-p", type=str, required=False, default="grype*.json")
+def import_scans(
+    destination: Optional[Path],
+    file_pattern: str,
+    folder_path: Path,
+    regscale_ssp_id: int,
+    scan_date: datetime,
+    mappings_path: Path,
+    disable_mapping: bool,
+    s3_bucket: str,
+    s3_prefix: str,
+    aws_profile: str,
+    upload_file: bool,
+) -> None:
+    """
+    Process Grype scan results from a folder containing Grype scan files and load into RegScale.
+    """
+    import_grype_scans(
+        destination=destination,
+        file_pattern=file_pattern,
+        folder_path=folder_path,
+        regscale_ssp_id=regscale_ssp_id,
+        scan_date=scan_date,
+        mappings_path=mappings_path,
+        disable_mapping=disable_mapping,
+        s3_bucket=s3_bucket,
+        s3_prefix=s3_prefix,
+        aws_profile=aws_profile,
+        upload_file=upload_file,
+    )
+
+
+def import_grype_scans(
+    folder_path: Path,
+    regscale_ssp_id: int,
+    scan_date: datetime,
+    mappings_path: Optional[Path] = None,
+    disable_mapping: Optional[bool] = False,
+    s3_bucket: Optional[str] = None,
+    s3_prefix: Optional[str] = None,
+    aws_profile: Optional[str] = None,
+    destination: Optional[Path] = None,
+    file_pattern: Optional[str] = "grype*.json",
+    upload_file: Optional[bool] = True,
+) -> None:
+    """
+    Process Grype scan results from a folder container grype scan files and load into RegScale.
+
+    :param Path folder_path: Path to the Grype scan results JSON file
+    :param int regscale_ssp_id: RegScale SSP ID
+    :param datetime scan_date: The date of the scan
+    :param Optional[Path] mappings_path: Path to the header mapping file, default: None
+    :param Optional[bool] disable_mapping: Disable the header mapping, default: False
+    :param Optional[str] s3_bucket: S3 bucket to download scan files from, default: None
+    :param Optional[str] s3_prefix: Prefix (folder path) within the S3 bucket, default: None
+    :param Optional[str] aws_profile: AWS profile to use for S3 access, default: None
+    :param Optional[Path] destination: Destination folder for processed files, default: None
+    :param Optional[str] file_pattern: File pattern to search for in the directory, default: grype*.json
+    :param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, default: True
+    :raises GrypeProcessingError: If there is an error processing the Grype results
+    :rtype: None
+    """
+    from regscale.exceptions import ValidationException
+    from regscale.core.app.application import Application
+
+    try:
+        if s3_bucket and s3_prefix and aws_profile:
+            download_from_s3(bucket=s3_bucket, prefix=s3_prefix, local_path=destination, aws_profile=aws_profile)
+            files = find_files(path=destination, pattern=file_pattern)
+            logger.info("Downloaded all Grype scan files from S3. Processing...")
+        elif destination and not s3_bucket:
+            logger.info("Moving Grype scan files to %s", destination)
+            stored_file_collection = find_files(path=folder_path, pattern=file_pattern)
+            move_all_files(stored_file_collection, destination)
+            files = find_files(path=destination, pattern=file_pattern)
+            logger.info("Done moving files")
+        else:
+            stored_file_collection = find_files(path=folder_path, pattern=file_pattern)
+            files = stored_file_collection
+        if not files:
+            logger.error("No Grype scan results found in the specified directory")
+            return
+
+    except Exception as e:
+        logger.error(f"Error processing Grype results: {str(e)}")
+        logger.error(traceback.format_exc())
+        raise GrypeProcessingError(f"Failed to process Grype results: {str(e)}")
+
+    for file in files:
+        try:
+            GrypeImport(
+                name="Grype",
+                app=Application(),
+                file_path=str(file),
+                file_type=file.suffix,
+                parent_id=regscale_ssp_id,
+                parent_module="securityplans",
+                scan_date=scan_date,
+                mappings_path=mappings_path,
+                disable_mapping=disable_mapping,
+                upload_file=upload_file,
+                file_name=file.name,
+            )
+        except ValidationException as e:
+            logger.error(f"Validation error on {file}: {e}")
+            continue
+    logger.info("Completed Grype processing.")
+
+
+def move_all_files(file_collection: List[Union[Path, str]], destination: Union[Path, str]) -> None:
+    """
+    Move all Grype files in the current directory to a folder called 'processed'.
+
+    :param List[Union[Path, str]] file_collection: A list of file paths or S3 URIs
+    :param Union[Path, str] destination: The destination folder
+    :rtype: None
+    """
+    for file in iterate_files(file_collection):
+        file_path = Path(file)
+        new_filename = f"{file_path.stem}{file_path.suffix}"
+        new_file_path = Path(destination) / new_filename
+        move_file(file, new_file_path)

regscale/integrations/commercial/ibm.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""IBM AppScan RegScale integration"""
+from datetime import datetime
+from os import PathLike
+from typing import Optional
+
+import click
+
+from regscale.models.integration_models.flat_file_importer import FlatFileImporter
+from regscale.models.integration_models.ibm import AppScan
+
+
+@click.group()
+def ibm():
+    """Performs actions on IBM AppScan files."""
+
+
+@ibm.command(name="import_appscan")
+@FlatFileImporter.common_scanner_options(
+    message="File path to the folder containing IBM AppScan .csv files to process to RegScale.",
+    prompt="File path for IBM AppScan files",
+    import_name="ibm_appscan",
+)
+def import_appscan(
+    folder_path: PathLike[str],
+    regscale_ssp_id: int,
+    scan_date: datetime,
+    mappings_path: PathLike[str],
+    disable_mapping: bool,
+    s3_bucket: str,
+    s3_prefix: str,
+    aws_profile: str,
+    upload_file: bool,
+) -> None:
+    """
+    Import IBM AppScan scans, vulnerabilities and assets to RegScale from IBM AppScan files
+    """
+    import_appscan_files(
+        folder_path,
+        regscale_ssp_id,
+        scan_date,
+        mappings_path,
+        disable_mapping,
+        s3_bucket,
+        s3_prefix,
+        aws_profile,
+        upload_file,
+    )
+
+
+def import_appscan_files(
+    folder_path: PathLike[str],
+    regscale_ssp_id: int,
+    scan_date: datetime,
+    mappings_path: PathLike[str],
+    disable_mapping: bool,
+    s3_bucket: str,
+    s3_prefix: str,
+    aws_profile: str,
+    upload_file: Optional[bool] = True,
+) -> None:
+    """
+    Import IBM AppScan scans, vulnerabilities and assets to RegScale from IBM AppScan files
+
+    :param PathLike[str] folder_path: File path to the folder containing IBM AppScan .csv files to process to RegScale
+    :param int regscale_ssp_id: The RegScale SSP ID
+    :param datetime scan_date: The date of the scan
+    :param PathLike[str] mappings_path: The path to the mappings file
+    :param bool disable_mapping: Whether to disable custom mappings
+    :param str s3_bucket: The S3 bucket to download the files from
+    :param str s3_prefix: The S3 prefix to download the files from
+    :param str aws_profile: The AWS profile to use for S3 access
+    :param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True
+    :rtype: None
+    """
+    FlatFileImporter.import_files(
+        import_type=AppScan,
+        import_name="IBM AppScan",
+        file_types=".csv",
+        folder_path=folder_path,
+        regscale_ssp_id=regscale_ssp_id,
+        scan_date=scan_date,
+        mappings_path=mappings_path,
+        disable_mapping=disable_mapping,
+        s3_bucket=s3_bucket,
+        s3_prefix=s3_prefix,
+        aws_profile=aws_profile,
+        upload_file=upload_file,
+    )