regscale-cli 6.16.0.0__py3-none-any.whl → 6.16.2.0__py3-none-any.whl

regscale/integrations/commercial/sap/sysdig/sysdig_scanner.py

@@ -74,9 +74,11 @@ class SAPConcurSysDigScanner(ScannerIntegration):
             raise ValueError("Path is required")
 
         logger.info(f"Fetching assets from {path}")
+        self.num_assets_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_assets_to_process += 1
                 yield self.parse_assets(row)
 
     def fetch_findings(self, *args: Tuple, **kwargs: dict) -> Iterator[IntegrationFinding]:
@@ -94,9 +96,11 @@ class SAPConcurSysDigScanner(ScannerIntegration):
 
         logger.info(f"Fetching findings from {path}")
 
+        self.num_findings_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_findings_to_process += 1
                 yield from self.parse_findings(finding=row, kwargs=kwargs)
 
     def parse_findings(self, finding: Dict[str, Any], **kwargs: dict) -> Iterator[IntegrationFinding]:

regscale/integrations/commercial/sap/tenable/click.py

@@ -16,7 +16,7 @@ logger = logging.getLogger("regscale")
 @regscale_ssp_id()
 @click.option(
     "--path",
-    type=click.STRING,
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
     help="Path to the CSV file containing the SAP Concur data.",
     required=True,
 )

regscale/integrations/commercial/sap/tenable/scanner.py

@@ -30,7 +30,8 @@ class SAPConcurScanner(ScannerIntegration):
         "low": regscale_models.IssueSeverity.Low,
     }
 
-    def parse_assets(self, asset: Dict[str, Any]) -> IntegrationAsset:
+    @staticmethod
+    def parse_assets(asset: Dict[str, Any]) -> IntegrationAsset:
         """
         Parse a single asset from the vulnerability data.
 
@@ -88,9 +89,11 @@ class SAPConcurScanner(ScannerIntegration):
             raise ValueError("Path is required")
 
         logger.info(f"Fetching assets from {path}")
+        self.num_assets_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_assets_to_process += 1
                 yield self.parse_assets(row)
 
     def fetch_findings(self, *args: Tuple, **kwargs: dict) -> Iterator[IntegrationFinding]:
@@ -108,9 +111,11 @@ class SAPConcurScanner(ScannerIntegration):
 
         logger.info(f"Fetching findings from {path}")
 
+        self.num_findings_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_assets_to_process += 1
                 yield from self.parse_findings(row)
 
     def parse_findings(self, finding: Dict[str, Any]) -> Iterator[IntegrationFinding]:
@@ -132,8 +137,9 @@ class SAPConcurScanner(ScannerIntegration):
             for cve in cves:
                 yield self._create_finding(finding, severity, cve.strip())
 
+    @staticmethod
     def _create_finding(
-        self, finding: Dict[str, Any], severity: regscale_models.IssueSeverity, cve: str = ""
+        finding: Dict[str, Any], severity: regscale_models.IssueSeverity, cve: str = ""
     ) -> IntegrationFinding:
         """
         Create an IntegrationFinding object from the given data.

regscale/integrations/commercial/tenablev2/click.py

@@ -3,6 +3,7 @@
 """Tenable integration for RegScale CLI"""
 
 import queue
+from concurrent.futures import wait
 from typing import TYPE_CHECKING, Any
 
 from regscale.integrations.integration_override import IntegrationOverride
@@ -723,16 +724,20 @@ def fetch_vulns(query_id: int = 0, regscale_ssp_id: int = 0):
         # make sure folder exists
         with tempfile.TemporaryDirectory() as temp_dir:
             logger.info("Saving Tenable SC data to disk...%s", temp_dir)
-            consume_iterator_to_file(iterator=vulns, dir_path=Path(temp_dir), scanner=sc)
+            num_assets_processed, num_findings_to_process = consume_iterator_to_file(
+                iterator=vulns, dir_path=Path(temp_dir), scanner=sc
+            )
             iterables = tenable_dir_to_tuple_generator(Path(temp_dir))
             try:
                 sc.sync_assets(
                     plan_id=regscale_ssp_id,
                     integration_assets=(asset for sublist in iterables[0] for asset in sublist),
+                    asset_count=num_assets_processed,
                 )
                 sc.sync_findings(
                     plan_id=regscale_ssp_id,
                     integration_findings=(finding for sublist in iterables[1] for finding in sublist),
+                    finding_count=num_findings_to_process,
                 )
             except IndexError as ex:
                 logger.error("Error processing Tenable SC data: %s", ex)
@@ -754,23 +759,26 @@ def tenable_dir_to_tuple_generator(dir_path: Path):
     return assets_gen, findings_gen
 
 
-def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path, scanner: SCIntegration) -> int:
+def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path, scanner: SCIntegration) -> tuple:
     """
     Consume an iterator and write the results to a file
 
     :param AnalysisResultsIterator iterator: Tenable SC iterator
     :param Path dir_path: The directory to save the pickled files
     :param SCIntegration scanner: Tenable SC Integration object
-    :rtype: int
-    :return: The total count of items processed
+    :rtype: tuple
+    :return: The total count of assets and findings processed
     """
     app = Application()
     logger.info("Consuming Tenable SC iterator...")
     override = IntegrationOverride(app)
+    asset_count = 0
+    findings_count = 0
     total_count = ThreadSafeCounter()
     page_number = ThreadSafeCounter()
     rec_count = ThreadSafeCounter()
     process_list = queue.Queue()
+    futures_lst = []
     with ThreadPoolExecutor(max_workers=5) as executor:
         for dat in iterator:
             total_count.increment()
@@ -778,19 +786,32 @@ def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path,
             rec_count.increment()
             if rec_count.value == len(iterator.page):
                 page_number.increment()
-                executor.submit(
-                    process_sc_chunk,
-                    app=app,
-                    vulns=pop_queue(queue=process_list, queue_len=len(iterator.page)),
-                    page=page_number.value,
-                    dir_path=dir_path,
-                    sc=scanner,
-                    override=override,
+                futures_lst.append(
+                    executor.submit(
+                        process_sc_chunk,
+                        app=app,
+                        vulns=pop_queue(queue=process_list, queue_len=len(iterator.page)),
+                        page=page_number.value,
+                        dir_path=dir_path,
+                        sc=scanner,
+                        override=override,
+                    )
                 )
                 rec_count.set(0)
+    # Collect results from all threads
+    asset_count = 0
+    findings_count = 0
+    # Wait for completion
+    wait(futures_lst)
+
+    for future in futures_lst:
+        findings, assets = future.result()
+        asset_count += assets
+        findings_count += findings
+
     if total_count.value == 0:
         logger.warning("No Tenable SC data found.")
-    return total_count.value
+    return asset_count, findings_count
 
 
 def pop_queue(queue: queue.Queue, queue_len: int) -> list:
@@ -816,12 +837,13 @@ def pop_queue(queue: queue.Queue, queue_len: int) -> list:
     return retrieved_items
 
 
-def process_sc_chunk(**kwargs) -> None:
+def process_sc_chunk(**kwargs) -> tuple:
     """
     Process Tenable SC chunk
 
     :param kwargs: Keyword arguments
-    :rtype: None
+    :rtype: tuple
+    :return: Tuple of findings and assets
     """
     # iterator.page, iterator.page_count, file_path, query_id, ssp_id
     integration_mapping = kwargs.get("override")
@@ -833,7 +855,7 @@ def process_sc_chunk(**kwargs) -> None:
     tenable_sc: SCIntegration = kwargs.get("sc")
     thread = current_thread()
     if not len(vulns):
-        return
+        return (0, 0)
     # I can't add a to-do thanks to sonarlint, but we need to add CVE lookup from plugin id
     # append file to path
     # Process to RegScale
@@ -857,6 +879,7 @@ def process_sc_chunk(**kwargs) -> None:
         kwargs.get("page"),
     )
     logger.debug(f"Completed thread: name={thread.name}, idnet={get_ident()}, id={get_native_id()}")
+    return (len(findings), len(assets))
 
 
 def get_last_pull_epoch(regscale_ssp_id: int) -> int:

regscale/integrations/commercial/trivy/__init__.py

@@ -0,0 +1,5 @@
+"""
+Module for processing Trivy scan results and loading them into RegScale.
+"""
+
+from .commands import import_scans

regscale/integrations/commercial/trivy/commands.py

@@ -0,0 +1,74 @@
+"""
+This module contains the command line interface for the Trivy scanner integration.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+import click
+from pathlib import Path
+
+from regscale.models.integration_models.flat_file_importer import FlatFileImporter
+
+
+@click.group()
+def trivy():
+    """Performs actions from the Trivy scanner integration."""
+    pass
+
+
+@trivy.command("import_scans")
+@FlatFileImporter.common_scanner_options(
+    message="File path to the folder containing JFrog XRay .json files to process to RegScale.",
+    prompt="File path for Grype files",
+    import_name="grype",
+)
+@click.option(
+    "--destination",
+    "-d",
+    help="Path to download the files to. If not provided, files will be downloaded to the temporary directory.",
+    type=click.Path(exists=True, dir_okay=True),
+    required=False,
+)
+@click.option(
+    "--file_pattern",
+    "-fp",
+    help="[Optional] File pattern to match (e.g., '*.json')",
+    required=False,
+)
+def import_scans(
+    destination: Optional[Path],
+    file_pattern: str,
+    folder_path: Path,
+    regscale_ssp_id: int,
+    scan_date: datetime,
+    mappings_path: Path,
+    disable_mapping: bool,
+    s3_bucket: str,
+    s3_prefix: str,
+    aws_profile: str,
+    upload_file: bool,
+) -> None:
+    """
+    Process Trivy scan results from a folder containing Trivy scan files and load into RegScale.
+    """
+    from regscale.integrations.commercial.trivy.scanner import TrivyIntegration
+
+    if s3_bucket and not folder_path:
+        folder_path = s3_bucket
+    ti = TrivyIntegration(
+        plan_id=regscale_ssp_id,
+        file_path=str(folder_path) if folder_path else None,
+        s3_bucket=s3_bucket,
+        s3_prefix=s3_prefix,
+        aws_profile=aws_profile,
+        scan_date=scan_date,
+        mappings_path=str(mappings_path) if mappings_path else None,
+        disable_mapping=disable_mapping,
+        download_destination=destination,
+        file_pattern=file_pattern,
+        read_files_only=True,
+        upload_file=upload_file,
+    )
+
+    ti.sync_assets_and_findings()

regscale/integrations/commercial/trivy/scanner.py

@@ -0,0 +1,276 @@
+"""
+Trivy Scanner Integration for RegScale.
+
+This module provides integration between Trivy scanner and RegScale,
+allowing you to import Trivy scan results into RegScale as assets and findings.
+"""
+
+import logging
+import os
+from typing import Any, Dict, List, Optional, Union, Tuple, TypeVar
+
+from pathlib import Path
+
+from regscale.core.app.utils.parser_utils import safe_datetime_str
+from regscale.integrations.jsonl_scanner_integration import JSONLScannerIntegration
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
+from regscale.models import IssueSeverity, AssetStatus, IssueStatus
+
+logger = logging.getLogger("regscale")
+
+# Define generic types for items that can be written to file
+T = TypeVar("T")
+ItemType = TypeVar("ItemType", IntegrationAsset, IntegrationFinding)
+
+
+class TrivyIntegration(JSONLScannerIntegration):
+    """Class for handling Trivy scanner integration."""
+
+    title: str = "Trivy"
+    asset_identifier_field: str = "otherTrackingNumber"
+    finding_severity_map: Dict[str, Any] = {
+        "CRITICAL": IssueSeverity.Critical.value,
+        "HIGH": IssueSeverity.High.value,
+        "MEDIUM": IssueSeverity.Moderate.value,
+        "LOW": IssueSeverity.Low.value,
+        "UNKNOWN": IssueSeverity.High.value,
+        "NEGLIGIBLE": IssueSeverity.High.value,
+    }
+
+    # Constants for file paths
+    ASSETS_FILE = "./artifacts/trivy_assets.jsonl"
+    FINDINGS_FILE = "./artifacts/trivy_findings.jsonl"
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the TrivyIntegration object.
+
+        :param Any kwargs: Keyword arguments
+        """
+        kwargs["read_files_only"] = True
+        kwargs["file_pattern"] = "*.json"
+        self.disable_mapping = kwargs["disable_mapping"] = True
+        super().__init__(*args, **kwargs)
+
+    def is_valid_file(self, data: Any, file_path: Union[Path, str]) -> Tuple[bool, Optional[Dict[str, Any]]]:
+        """
+        Check if the provided data is a valid Trivy scan result.
+
+        Validates that the data is from a Trivy JSON file with the required structure.
+        Logs a warning with the file path and returns (False, None) if invalid.
+
+        :param Any data: Data parsed from the file (string content when read_files_only is True, or file path otherwise)
+        :param Union[Path, str] file_path: Path to the file being processed
+        :return: Tuple of (is_valid, validated_data) where validated_data is the parsed JSON if valid
+        :rtype: Tuple[bool, Optional[Dict[str, Any]]]
+        """
+
+        # Check Trivy-specific structure
+        if not isinstance(data, dict):
+            logger.warning(f"File {file_path} is not a dict, skipping")
+            return False, None
+
+        if "Results" not in data:
+            logger.warning(f"File {file_path} has no 'Results' key, skipping")
+            return False, None
+
+        if not isinstance(data.get("Results"), list):
+            logger.warning(f"File {file_path} 'Results' is not a list, skipping")
+            return False, None
+
+        if "Metadata" not in data:
+            logger.warning(f"File {file_path} has no 'Metadata' key, skipping")
+            return False, None
+
+        return True, data
+
+    def parse_asset(self, file_path: Union[Path, str], data: Dict[str, Any]) -> IntegrationAsset:
+        """
+        Parse a single asset from Trivy scan data.
+
+        :param Union[Path, str] file_path: Path to the file containing the asset data
+        :param Dict[str, Any] data: The parsed JSON data
+        :return: IntegrationAsset object
+        :rtype: IntegrationAsset
+        """
+        # Convert path to string if it's not already
+        file_path_str = str(file_path)
+
+        # Get metadata and OS information
+        metadata = data.get("Metadata", {})
+        os_data = metadata.get("OS", {})
+
+        # Determine identifier from file name or data
+        if "sha256-" in file_path_str:
+            # Extract the sha256 from the filename
+            base_name = os.path.basename(file_path_str)
+            identifier = "sha256-" + base_name.split("sha256-")[1].split(".json")[0]
+        else:
+            identifier = metadata.get("ImageID", "Unknown")
+
+        # Get artifact name for other tracking number and fqdn
+        artifact_name = data.get("ArtifactName", identifier)
+
+        # Create and return the asset
+        return IntegrationAsset(
+            identifier=identifier,
+            name=identifier,
+            ip_address="0.0.0.0",
+            cpu=0,
+            ram=0,
+            status=AssetStatus.Active,
+            asset_type="Other",
+            asset_category="Software",
+            operating_system=f"{os_data.get('Family', '')} {os_data.get('Name', '')}",
+            notes=f"{os.path.basename(file_path_str)}",
+            other_tracking_number=artifact_name,
+            parent_id=self.plan_id,
+            parent_module="securityplans",
+            fqdn=artifact_name,
+        )
+
+    def _get_findings_data_from_file(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Extract findings data from Trivy file data.
+
+        :param Dict[str, Any] data: The data from the Trivy file
+        :return: List of finding items
+        :rtype: List[Dict[str, Any]]
+        """
+        if not data or not isinstance(data, dict):
+            return []
+
+        findings = []
+
+        # Process all results
+        for result in data.get("Results", []):
+            if not isinstance(result, dict):
+                continue
+
+            # Extract vulnerabilities from the result
+            vulnerabilities = result.get("Vulnerabilities", [])
+            if not isinstance(vulnerabilities, list):
+                continue
+
+            # Add each vulnerability to the findings list
+            findings.extend(vulnerabilities)
+
+        return findings
+
+    def parse_finding(self, asset_identifier: str, data: Dict[str, Any], item: Dict[str, Any]) -> IntegrationFinding:
+        """
+        Parse a single finding from Trivy scan data.
+
+        :param str asset_identifier: The identifier of the asset this finding belongs to
+        :param Dict[str, Any] data: The parsed JSON data (for metadata)
+        :param Dict[str, Any] item: The finding data
+        :return: IntegrationFinding object
+        :rtype: IntegrationFinding
+        """
+        # Get scan date from the finding or use current time
+        scan_date = safe_datetime_str(data.get("CreatedAt"))
+
+        # Process severity
+        severity_str = item.get("Severity", "UNKNOWN")
+        severity_value = self.finding_severity_map.get(severity_str.upper(), IssueSeverity.High.value)
+        try:
+            severity = IssueSeverity(severity_value)
+        except ValueError:
+            severity = IssueSeverity.High
+
+        # Get CVSS fields
+        cvss_fields = self._get_cvss_score(item)
+
+        # Get data source information
+        data_source = item.get("DataSource", {})
+        plugin_name = data_source.get("Name", self.title)
+        plugin_id = data_source.get("ID", self.title)
+
+        metadata = data.get("Metadata", {})
+        os_family = metadata.get("OS", {}).get("Family", "")
+        os_name = metadata.get("OS", {}).get("Name", "")
+        if os_family and os_name == "unknown":
+            affected_os = "unknown"
+        else:
+            affected_os = f"{os_family} {os_name}"
+
+        # Set image digest from artifact name
+        artifact_name = data.get("ArtifactName", "")
+        image_digest = ""
+        if "@" in artifact_name:
+            image_digest = artifact_name.split("@")[1]
+
+        build_version = (
+            metadata.get("ImageConfig", {}).get("config", {}).get("Labels", {}).get("io.buildah.version", "")
+        )
+        pkg_name = item.get("PkgName", "")
+        cve = item.get("VulnerabilityID", "")
+
+        # Create and return the finding
+        return IntegrationFinding(
+            title=f"{cve}: {pkg_name}" if cve else pkg_name,
+            description=item.get("Description", "No description available"),
+            severity=severity,
+            status=IssueStatus.Open,
+            cvss_v3_score=cvss_fields.get("V3Score"),
+            cvss_v3_vector=cvss_fields.get("V3Vector") or "",
+            cvss_v2_score=cvss_fields.get("V2Score"),
+            cvss_v2_vector=cvss_fields.get("V2Vector") or "",
+            plugin_name=plugin_name,
+            plugin_id=plugin_id,
+            asset_identifier=asset_identifier,
+            cve=cve,
+            first_seen=scan_date,
+            last_seen=scan_date,
+            scan_date=scan_date,
+            date_created=scan_date,
+            category="Software",
+            control_labels=[],
+            installed_versions=item.get("InstalledVersion", ""),
+            affected_os=affected_os,
+            affected_packages=item.get("PkgID", ""),
+            image_digest=image_digest,
+            package_path=item.get("PkgIdentifier", {}).get("PURL", ""),
+            build_version=build_version,
+            fixed_versions=item.get("FixedVersion", ""),
+            fix_status=item.get("Status", ""),
+        )
+
+    @staticmethod
+    def _get_cvss_score(finding: Dict) -> dict:
+        """
+        Get the CVSS v3 and v2 scores and vectors from the cvss data.
+
+        :param Dict finding: The cvss data
+        :return: The CVSS fields
+        :rtype: dict
+        """
+        values = {
+            "V3Score": None,
+            "V2Score": None,
+            "V3Vector": None,
+            "V2Vector": None,
+        }
+
+        if cvs := finding.get("CVSS"):
+            if nvd := cvs.get("nvd"):
+                values["V3Score"] = nvd.get("V3Score", None)
+                values["V3Vector"] = nvd.get("V3Vector", None)
+                values["V2Score"] = nvd.get("V2Score", None)
+                values["V2Vector"] = nvd.get("V2Vector", None)
+            elif redhat := cvs.get("redhat"):
+                values["V3Score"] = redhat.get("V3Score", None)
+                values["V3Vector"] = redhat.get("V3Vector", None)
+                values["V2Score"] = redhat.get("V2Score", None)
+                values["V2Vector"] = redhat.get("V2Vector", None)
+            elif ghsa := cvs.get("ghsa"):
+                values["V3Score"] = ghsa.get("V3Score", None)
+                values["V3Vector"] = ghsa.get("V3Vector", None)
+                values["V2Score"] = ghsa.get("V2Score", None)
+                values["V2Vector"] = ghsa.get("V2Vector", None)
+            elif bitnami := cvs.get("bitnami"):
+                values["V3Score"] = bitnami.get("V3Score", None)
+                values["V3Vector"] = bitnami.get("V3Vector", None)
+                values["V2Score"] = bitnami.get("V2Score", None)
+                values["V2Vector"] = bitnami.get("V2Vector", None)
+        return values

regscale/integrations/commercial/wizv2/click.py

@@ -394,13 +394,6 @@ def add_report_evidence(
     default="NIST800-53R5",
     required=True,
 )
-@click.option(  # type: ignore
-    "--include_not_implemented",
-    "-n",
-    is_flag=True,
-    help="Include not implemented controls",
-    default=False,
-)
 def sync_compliance(
     wiz_project_id,
     regscale_id,
@@ -409,21 +402,16 @@ def sync_compliance(
     client_secret,
     catalog_id,
     framework,
-    include_not_implemented,
 ):
     """Sync compliance posture from Wiz to RegScale"""
-    from regscale.core.app.utils.app_utils import create_progress_object
     from regscale.integrations.commercial.wizv2.utils import _sync_compliance
 
-    compliance_job_progress = create_progress_object()
-    with compliance_job_progress:
-        _sync_compliance(
-            wiz_project_id=wiz_project_id,
-            regscale_id=regscale_id,
-            regscale_module=regscale_module,
-            include_not_implemented=include_not_implemented,
-            client_id=client_id,
-            client_secret=client_secret,
-            catalog_id=catalog_id,
-            framework=framework,
-        )
+    _sync_compliance(
+        wiz_project_id=wiz_project_id,
+        regscale_id=regscale_id,
+        regscale_module=regscale_module,
+        client_id=client_id,
+        client_secret=client_secret,
+        catalog_id=catalog_id,
+        framework=framework,
+    )

regscale/integrations/commercial/wizv2/scanner.py

@@ -101,6 +101,7 @@ class WizVulnerabilityIntegration(ScannerIntegration):
             raise ValueError("Wiz project ID is required")
 
         logger.info("Fetching Wiz findings...")
+        self.num_findings_to_process = 0
 
         for wiz_vulnerability_type in self.get_query_types(project_id=project_id):
             logger.info("Fetching Wiz findings for %s...", wiz_vulnerability_type["type"])
@@ -114,8 +115,8 @@ class WizVulnerabilityIntegration(ScannerIntegration):
                 topic_key=wiz_vulnerability_type["topic_key"],
                 file_path=wiz_vulnerability_type["file_path"],
             )
+            self.num_findings_to_process += len(nodes)
             yield from self.parse_findings(nodes, wiz_vulnerability_type["type"])
-
         logger.info("Finished fetching Wiz findings.")
 
     def parse_findings(