regscale-cli 6.20.6.0__py3-none-any.whl → 6.20.8.0__py3-none-any.whl

regscale/models/integration_models/flat_file_importer/__init__.py

@@ -1,14 +1,13 @@
 """Container Scan Abstract"""
 
 import ast
-import csv
 import json
 import logging
 import re
 import shutil
 from abc import ABC, abstractmethod
 from collections import namedtuple
-from datetime import datetime, timedelta
+from datetime import datetime
 from os import PathLike
 from typing import TYPE_CHECKING, Any, Callable, Generator, Iterator, List, Optional, Sequence, TextIO, Tuple, Union
 
@@ -18,7 +17,6 @@ if TYPE_CHECKING:
 from pathlib import Path
 
 import click
-import requests
 import xmltodict
 from openpyxl.reader.excel import load_workbook
 
@@ -32,11 +30,10 @@ from regscale.core.app.utils.app_utils import (
     get_current_datetime,
 )
 from regscale.core.app.utils.parser_utils import safe_datetime_str
-from regscale.core.app.utils.report_utils import ReportGenerator
 from regscale.integrations.scanner_integration import ScannerIntegration
 from regscale.models import IssueStatus, Metadata, regscale_models
 from regscale.models.app_models.mapping import Mapping
-from regscale.models.regscale_models import Asset, File, Issue, Vulnerability
+from regscale.models.regscale_models import Asset, File, Vulnerability
 
 logger = logging.getLogger(__name__)
 
@@ -74,6 +71,7 @@ class FlatFileIntegration(ScannerIntegration):
                 "Low": regscale_models.IssueSeverity.Low,
             }
         super().__init__(plan_id=plan_id, **kwargs)
+        self.is_component = kwargs.get("is_component", False)
 
     def set_asset_identifier_field(self, asset_identifier_field: str) -> None:
         """
@@ -130,10 +128,16 @@ class FlatFileImporter(ABC):
                 "Medium": regscale_models.IssueSeverity.Moderate,
                 "Low": regscale_models.IssueSeverity.Low,
             }
-        if "parent_id" not in kwargs and "regscale_ssp_id" in kwargs:
-            kwargs["parent_id"] = kwargs["regscale_ssp_id"]
-            kwargs["parent_module"] = "securityplans"
-            kwargs["plan_id"] = kwargs["regscale_ssp_id"]
+
+        # Set the parent_id, parent_module, and plan_id if they are not provided
+        if "parent_id" not in kwargs and "object_id" in kwargs:
+            kwargs["parent_id"] = kwargs["object_id"]
+            kwargs["plan_id"] = kwargs["object_id"]
+        if kwargs.get("is_component", False):
+            kwargs["parent_module"] = regscale_models.Component.get_module_string()
+        else:
+            kwargs["parent_module"] = regscale_models.SecurityPlan.get_module_string()
+
         if "app" not in kwargs:
             kwargs["app"] = Application()
         # empty generator
@@ -156,7 +160,8 @@ class FlatFileImporter(ABC):
             "mapping",
             "ignore_validation",
             "header_line_number",
-            "regscale_ssp_id",
+            "is_component",
+            "object_id",
             "plan_id",
             "disable_mapping",
             "mappings_path",
@@ -188,7 +193,8 @@ class FlatFileImporter(ABC):
         }
         self.create_epoch = str(int(creation_date(self.attributes.file_path)))
         flat_int = FlatFileIntegration(
-            plan_id=self.attributes.plan_id or self.attributes.regscale_ssp_id or self.attributes.parent_id,
+            plan_id=self.attributes.plan_id or self.attributes.object_id or self.attributes.parent_id,
+            is_component=self.attributes.is_component,
             asset_identifier_field=self.asset_identifier_field,
             finding_severity_map=self.finding_severity_map,
         )
@@ -206,13 +212,15 @@ class FlatFileImporter(ABC):
         elif isinstance(self.data["vulns"], list):
             flat_int.num_findings_to_process = len(self.data["vulns"])
         flat_int.sync_assets(
-            plan_id=self.attributes.parent_id,
+            plan_id=self.attributes.plan_id,
+            is_component=self.attributes.is_component,
             integration_assets=self.integration_assets,
             title=self.attributes.name,
             asset_count=flat_int.num_assets_to_process,
         )
         flat_int.sync_findings(
-            plan_id=self.attributes.parent_id,
+            plan_id=self.attributes.plan_id,
+            is_component=self.attributes.is_component,
             integration_findings=self.integration_findings,
             title=self.attributes.name,
             finding_count=flat_int.num_findings_to_process,
@@ -319,7 +327,7 @@ class FlatFileImporter(ABC):
             asset_type=asset.assetType,
             asset_owner_id=asset.assetOwnerId,
             parent_id=self.attributes.parent_id,
-            parent_module=regscale_models.SecurityPlan.get_module_slug(),
+            parent_module=self.attributes.parent_module,
             asset_category=asset.assetCategory,
             date_last_updated=asset.dateLastUpdated,
             status=asset.status,
@@ -391,11 +399,10 @@ class FlatFileImporter(ABC):
             # Skip lines until the start line is reached
             for _ in range(start_line_number):
                 next(file)
-            if file.name.endswith(".csv"):
+            if file.name.endswith((".csv", ".xlsx")):
+                # Use the validater data for CSV and XLSX files to ensure proper mapping validation
                 data = self.validater.data.to_dict("records")
                 header = list(self.validater.parsed_headers)
-            elif file.name.endswith(".xlsx"):
-                data, header = self.convert_xlsx_to_dict(file, start_line_number)
             elif file.name.endswith(".json"):
                 try:
                     # Filter possible null values
@@ -652,7 +659,7 @@ class FlatFileImporter(ABC):
         import_name: str,
         file_types: Union[str, list[str]],
         folder_path: PathLike[str],
-        regscale_ssp_id: int,
+        object_id: int,
         scan_date: datetime,
         mappings_path: Union[PathLike[str], Path],
         disable_mapping: bool,
@@ -669,7 +676,7 @@ class FlatFileImporter(ABC):
         :param str import_name: The name of the import type
         :param Union[str, list[str]] file_types: The file types to glob and import, e.g. ".csv" or [".csv", ".xlsx"]
         :param PathLike[str] folder_path: The folder path to import from
-        :param int regscale_ssp_id: The RegScale SSP ID
+        :param int object_id: The RegScale SSP ID
         :param datetime scan_date: The date of the scan
         :param Union[PathLike[str], Path] mappings_path: The path to the mappings file
         :param bool disable_mapping: Whether to disable custom mappings
@@ -686,9 +693,16 @@ class FlatFileImporter(ABC):
         if s3_bucket:
             download_from_s3(s3_bucket, s3_prefix, folder_path, aws_profile)
         app = Application()
-        if not validate_regscale_object(regscale_ssp_id, "securityplans"):
-            app.logger.warning("SSP #%i is not a valid RegScale Security Plan.", regscale_ssp_id)
+
+        # Validate the parent_id is a valid RegScale object
+        is_component = kwargs.get("is_component", False)
+        if is_component and not validate_regscale_object(object_id, "components"):
+            app.logger.warning("Component #%i is not a valid RegScale Component.", object_id)
+            return
+        elif not is_component and not validate_regscale_object(object_id, "securityplans"):
+            app.logger.warning("SSP #%i is not a valid RegScale Security Plan.", object_id)
             return
+
         if not scan_date or not FlatFileImporter.check_date_format(scan_date):
             scan_date = datetime.now()
         if isinstance(file_types, str):
@@ -706,7 +720,7 @@ class FlatFileImporter(ABC):
                 import_type(
                     name=import_name,
                     file_path=str(file),
-                    regscale_ssp_id=regscale_ssp_id,
+                    object_id=object_id,
                     scan_date=scan_date,
                     mappings_path=mappings_path,
                     disable_mapping=disable_mapping,

regscale/models/integration_models/qualys.py

@@ -7,10 +7,12 @@ import logging
 
 # pylint: disable=C0415
 import re
+from calendar import firstweekday
 from datetime import datetime
 from typing import Any, Iterator, Optional, TypeVar, TextIO, Union
 
 from openpyxl.reader.excel import load_workbook
+from pandas import Timestamp
 
 from regscale.core.app import create_logger
 from regscale.core.app.application import Application
@@ -124,12 +126,27 @@ class Qualys(FlatFileImporter):
             }
         )
 
-    def create_vuln(self, dat: Optional[dict] = None, **kwargs) -> None:
+    def _convert_datetime_to_str(self, input_date: Union[Timestamp, datetime]) -> str:
+        """
+        Convert a datetime or Timestamp object to a string in the specified format.
+
+        :param Union[Timestamp, datetime] input_date: The date to convert.
+        :return: The date as a string in the format '%Y-%m-%d %H:%M:%S'.
+        :rtype: str
+        """
+        if isinstance(input_date, Timestamp):
+            input_date = input_date.to_pydatetime()
+        elif isinstance(input_date, str):
+            return input_date
+        return input_date.strftime(self.dt_format)
+
+    def create_vuln(self, dat: Optional[dict] = None, **kwargs) -> Optional[Vulnerability]:
         """
         Create a vuln from a row in the Qualys file
 
         :param Optional[dict] dat: Data row from CSV file, defaults to None
-        :rtype: None
+        :return: RegScale Vulnerability object or None
+        :rtype: Optional[Vulnerability]
         """
         from regscale.integrations.commercial.qualys import map_qualys_severity_to_regscale
 
@@ -145,6 +162,9 @@ class Qualys(FlatFileImporter):
         config = self.attributes.app.config
         asset_match = [asset for asset in self.data["assets"] if asset.name == dns]
         asset = asset_match[0] if asset_match else None
+        last_seen = self._convert_datetime_to_str(self.mapping.get_value(dat, "Last Detected"))
+        first_seen = self._convert_datetime_to_str(self.mapping.get_value(dat, "First Detected"))
+
         if dat and asset_match:
             regscale_vuln = Vulnerability(
                 id=0,
@@ -152,8 +172,8 @@ class Qualys(FlatFileImporter):
                 parentId=asset.id,
                 parentModule="assets",
                 ipAddress=self.mapping.get_value(dat, IP),
-                lastSeen=self.mapping.get_value(dat, "Last Detected"),
-                firstSeen=self.mapping.get_value(dat, "First Detected"),
+                lastSeen=last_seen,
+                firstSeen=first_seen,
                 daysOpen=None,
                 dns=self.mapping.get_value(dat, DNS, other_id),
                 mitigated=None,

regscale/models/integration_models/send_reminders.py

@@ -554,8 +554,14 @@ class SendReminders:
         )
 
         # update api pool limits to max_thread count from init.yaml
-        api.pool_connections = config["maxThreads"]
-        api.pool_maxsize = config["maxThreads"]
+        max_threads = config.get("maxThreads", 100)
+        if not isinstance(max_threads, int):
+            try:
+                max_threads = int(max_threads)
+            except (ValueError, TypeError):
+                max_threads = 100
+        api.pool_connections = max_threads
+        api.pool_maxsize = max_threads
 
         # get assigned threads
         for i in range(len(threads)):