regscale-cli 6.16.1.0__py3-none-any.whl → 6.16.3.0__py3-none-any.whl

regscale/models/integration_models/{flat_file_importer.py → flat_file_importer/__init__.py}

@@ -82,7 +82,7 @@ class FlatFileIntegration(ScannerIntegration):
         """
         self.asset_identifier_field = asset_identifier_field
 
-    def fetch_assets(self, *args: Any, **kwargs: Any) -> Iterator["IntegrationAsset"]:
+    def fetch_assets(self, *args: Tuple, **kwargs: dict) -> Iterator["IntegrationAsset"]:
         """
         Fetches assets from FlatFileImporter
 
@@ -220,16 +220,19 @@ class FlatFileImporter(ABC):
         )
         self.clean_up()
 
-    def parse_finding(self, vuln: Vulnerability) -> Optional["IntegrationFinding"]:
+    def parse_finding(self, vuln: Union[Vulnerability, "IntegrationFinding"]) -> Optional["IntegrationFinding"]:
         """
         Parses a vulnerability object into an IntegrationFinding object
 
-        :param Vulnerability vuln: A vulnerability object
+        :param Union[Vulnerability, IntegrationFinding] vuln: A vulnerability object
         :return: The parsed IntegrationFinding or None if parsing fails
         :rtype: Optional[IntegrationFinding]
         """
         from regscale.integrations.scanner_integration import IntegrationFinding
 
+        if isinstance(vuln, IntegrationFinding):
+            return vuln
+
         try:
             asset_id = vuln.dns or vuln.ipAddress
             if not asset_id:
@@ -237,9 +240,9 @@ class FlatFileImporter(ABC):
 
             severity = self.finding_severity_map.get(vuln.severity.capitalize(), regscale_models.IssueSeverity.Low)
             status = self.map_status_to_issue_status(vuln.status)
-            cve: Optional[str] = vuln.cve if vuln.cve else ""
-            extract_vuln: Any = self.extract_ghsa_strings(vuln.plugInName or "")
-            plugin_name = vuln.plugInName if vuln.plugInName else vuln.title
+            cve: Optional[str] = getattr(vuln, "cve", "")
+            extract_vuln: Any = self.extract_ghsa_strings(getattr(vuln, "plugInName", ""))
+            plugin_name = getattr(vuln, "plugInName", getattr(vuln, "title", ""))
             if not self.assert_valid_cve(cve):
                 if isinstance(extract_vuln, list):
                     cve = ", ".join(extract_vuln)
@@ -286,16 +289,19 @@ class FlatFileImporter(ABC):
             self.attributes.logger.error("Error parsing Wiz finding: %s", str(e), exc_info=True)
             return None
 
-    def parse_asset(self, asset: Asset) -> "IntegrationAsset":
+    def parse_asset(self, asset: Union[Asset, "IntegrationAsset"]) -> "IntegrationAsset":
         """
         Converts Asset -> IntegrationAsset
 
-        :param Asset asset: The asset to parse
+        :param Union[Asset, IntegrationAsset] asset: The asset to parse
         :return: The parsed IntegrationAsset
         :rtype: IntegrationAsset
         """
         from regscale.integrations.scanner_integration import IntegrationAsset
 
+        if isinstance(asset, IntegrationAsset):
+            return asset
+
         return IntegrationAsset(
             name=asset.name,
             external_id=asset.otherTrackingNumber,
@@ -514,6 +520,8 @@ class FlatFileImporter(ABC):
         """
         Process the assets in the data
         """
+        from regscale.integrations.scanner_integration import IntegrationAsset
+
         # The passed function creates asset objects. Convert to IntegrationAsset here
         if isinstance(self.file_data, list):
             for dat in self.file_data:
@@ -523,6 +531,8 @@ class FlatFileImporter(ABC):
         if isinstance(self.data["assets"], Iterator):
             self.integration_assets = self.data["assets"]
             return None
+        elif isinstance(self.data["assets"], IntegrationAsset):
+            self.data["assets"] = [self.data["assets"]]
         self.integration_assets = (self.parse_asset(asset) for asset in self.data["assets"])
 
     def process_asset_data(self, dat: Any, func: Callable) -> None:
@@ -712,6 +722,17 @@ class FlatFileImporter(ABC):
 
         with create_progress_object() as vuln_progress:
             vuln_task = vuln_progress.add_task("Processing vulnerabilities...", total=len(self.file_data))
+            try:
+                res = func(self.file_data)
+                if isinstance(res, list):
+                    self.integration_findings = res
+                    self.data["vulns"] = res
+                    vuln_progress.update(vuln_task, completed=len(self.file_data))
+                    return
+            except Exception as e:
+                self.attributes.logger.debug(
+                    "Cannot process vulns as a whole, now iterating all data to parse vulns: %s", str(e)
+                )
             for ix, dat in enumerate(self.file_data):
                 vuln = func(dat, index=ix)
                 if not vuln:

regscale/models/integration_models/snyk.py

@@ -3,15 +3,14 @@ Snyk Scan information
 """
 
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Union
 
 from regscale.core.app.application import Application
 from regscale.core.app.logz import create_logger
 from regscale.core.app.utils.app_utils import epoch_to_datetime, get_current_datetime, is_valid_fqdn
-from regscale.models import ImportValidater, Mapping
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
+from regscale.models import Asset, AssetCategory, AssetStatus, AssetType, ImportValidater, IssueStatus, Vulnerability
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
-from regscale.models.regscale_models.asset import Asset
-from regscale.models.regscale_models.vulnerability import Vulnerability
 
 
 class Snyk(FlatFileImporter):
@@ -20,19 +19,29 @@ class Snyk(FlatFileImporter):
     """
 
     def __init__(self, **kwargs):
+        self.not_implemented_error = "Unsupported file type for Snyk integration. Only XLSX and JSON are supported."
         self.name = kwargs.get("name")
-        self.vuln_title = "PROBLEM_TITLE"
-        self.project_name = "PROJECT_NAME"
-        self.issue_severity = "ISSUE_SEVERITY"
         self.auto_fixable = "AUTOFIXABLE"
         self.fmt = "%Y-%m-%d"
         self.dt_format = "%Y-%m-%d %H:%M:%S"
-        self.required_headers = [
-            self.project_name,
-            self.issue_severity,
-            self.vuln_title,
-            self.auto_fixable,
-        ]
+        if "json" in kwargs.get("file_type", ""):
+            self.project_name = "projectName"
+            self.issue_severity = "severity"
+            self.vuln_title = "title"
+            self.required_headers = [
+                "projectName",
+                "vulnerabilities",
+            ]
+        else:
+            self.project_name = "PROJECT_NAME"
+            self.issue_severity = "ISSUE_SEVERITY"
+            self.vuln_title = "PROBLEM_TITLE"
+            self.required_headers = [
+                self.project_name,
+                self.issue_severity,
+                self.vuln_title,
+                self.auto_fixable,
+            ]
         self.mapping_file = kwargs.get("mappings_path")
         self.disable_mapping = kwargs.get("disable_mapping")
         self.validater = ImportValidater(
@@ -40,6 +49,12 @@ class Snyk(FlatFileImporter):
         )
         self.headers = self.validater.parsed_headers
         self.mapping = self.validater.mapping
+        if "json" in kwargs.get("file_type", ""):
+            asset_count = 1
+            vuln_count = len(self.mapping.get_value(self.validater.data, "vulnerabilities", []))
+        else:
+            asset_count = None
+            vuln_count = None
         logger = create_logger()
         self.logger = logger
         super().__init__(
@@ -48,6 +63,8 @@ class Snyk(FlatFileImporter):
             headers=self.headers,
             asset_func=self.create_asset,
             vuln_func=self.create_vuln,
+            asset_count=asset_count,
+            vuln_count=vuln_count,
             extra_headers_allowed=True,
             **kwargs,
         )
@@ -65,7 +82,22 @@ class Snyk(FlatFileImporter):
             self.mapping.get_value(dat, "FIRST_INTRODUCED", datetime.now().time()),
         ).strftime(self.dt_format)
 
-    def create_asset(self, dat: Optional[dict] = None) -> Asset:
+    def create_asset(self, dat: Optional[dict] = None) -> Union[Asset, IntegrationAsset]:
+        """
+        Create an asset from a row in the Snyk file
+
+        :param Optional[dict] dat: Data row from XLSX file or JSON file, defaults to None
+        :return: RegScale Asset if XLSX, IntegrationAsset if JSON
+        :rtype: Union[Asset, IntegrationAsset]
+        """
+        if "json" in self.attributes.file_type:
+            return self._parse_json_asset(data=dat)
+        elif "xlsx" in self.attributes.file_type:
+            return self._parse_xlsx_asset(dat)
+        else:
+            raise NotImplementedError(self.not_implemented_error)
+
+    def _parse_xlsx_asset(self, dat: Optional[dict] = None) -> Asset:
         """
         Create an asset from a row in the Snyk file
 
@@ -95,7 +127,50 @@ class Snyk(FlatFileImporter):
             }
         )
 
-    def create_vuln(self, dat: Optional[dict] = None, **kwargs) -> Optional[Vulnerability]:
+    def _parse_json_asset(self, **kwargs) -> IntegrationAsset:
+        """
+        Parse assets from Snyk json scan data.
+
+        :return: Integration asset
+        :rtype: IntegrationAsset
+        """
+        data = kwargs.pop("data")
+        name = self.extract_host(self.mapping.get_value(data, self.project_name))
+        valid_name = is_valid_fqdn(name)
+        return IntegrationAsset(
+            identifier=name,
+            name=name,
+            status=AssetStatus.Active,
+            asset_category=AssetCategory.Software,
+            is_latest_scan=True,
+            is_authenticated_scan=True,
+            scanning_tool=self.name,
+            asset_type=AssetType.Other,
+            fqdn=name if valid_name else None,
+            system_administrator_id=self.config["userId"],
+            parent_id=self.attributes.parent_id,
+            parent_module=self.attributes.parent_module,
+        )
+
+    def create_vuln(
+        self, dat: Optional[dict] = None, **kwargs
+    ) -> Optional[Union[list[IntegrationFinding], Vulnerability]]:
+        """
+        Create a vulnerability from a row in the Snyk file
+
+        :param Optional[dict] dat: Data row from XLSX or JSON file, defaults to None
+        :raises TypeError: If dat is not a dictionary
+        :return: RegScale Vulnerability object if xlsx or list of IntegrationFindings if JSON
+        :rtype: Optional[Union[list[IntegrationFinding], Vulnerability]]
+        """
+        if "json" in self.attributes.file_type:
+            return self._parse_json_findings(**kwargs)
+        elif "xlsx" in self.attributes.file_type:
+            return self._parse_xlsx_finding(dat, **kwargs)
+        else:
+            raise NotImplementedError(self.not_implemented_error)
+
+    def _parse_xlsx_finding(self, dat: Optional[dict] = None, **_) -> Optional[Vulnerability]:
         """
         Create a vulnerability from a row in the Snyk csv file
 
@@ -139,6 +214,57 @@ class Snyk(FlatFileImporter):
             )
         return regscale_vuln
 
+    def _parse_json_findings(self, **kwargs) -> list[IntegrationFinding]:
+        """
+        Create a vulnerability from a row in the Snyk csv file
+
+        :return: List of IntegrationFinding objects
+        :rtype: list[IntegrationFinding]
+        """
+        findings = []
+        vulns = self.mapping.get_value(kwargs.get("data", self.validater.data), "vulnerabilities", [])
+        if not vulns:
+            return findings
+        for dat in vulns:
+            severity = self.finding_severity_map.get(dat.get(self.issue_severity, "Low").title())
+            hostname = self.extract_host(self.mapping.get_value(dat, self.project_name)) or self.extract_host(
+                self.mapping.get_value(self.validater.data, self.project_name)
+            )
+            description = self.mapping.get_value(dat, "description") or self.mapping.get_value(dat, self.vuln_title)
+            solution = self.mapping.get_value(dat, self.auto_fixable)
+            # if auto fixable is not available, check for upgradeable or patchable, this is for .json files
+            if not solution:
+                upgradeable = self.mapping.get_value(dat, "isUpgradeable", False)
+                patchable = self.mapping.get_value(dat, "isPatchable", False)
+                if upgradeable or patchable:
+                    solution = "Upgrade or patch the vulnerable component."
+            cves = ", ".join(self.mapping.get_value(dat, "CVE", ""))
+            if not cves:
+                cves = ", ".join(dat.get("identifiers", {}).get("CVE", []))
+            findings.append(
+                IntegrationFinding(
+                    title=dat.get("title") or description,
+                    description=description,
+                    severity=severity,
+                    status=IssueStatus.Open,
+                    plugin_name=description,
+                    plugin_id=dat.get("id"),
+                    plugin_text=self.mapping.get_value(dat, self.vuln_title),
+                    asset_identifier=hostname,
+                    cve=cves,
+                    cvss_score=dat.get("cvssScore"),
+                    first_seen=self.determine_first_seen(dat),
+                    last_seen=get_current_datetime(),
+                    scan_date=self.attributes.scan_date,
+                    dns=hostname,
+                    vpr_score=None,
+                    remediation=solution,
+                    category="Software",
+                    control_labels=[],
+                )
+            )
+        return findings
+
     @staticmethod
     def extract_host(s: str) -> str:
         """