regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

regscale/models/integration_models/synqly_models/ocsf_mapper.py

@@ -5,7 +5,11 @@ from typing import Any, Union, TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
     from regscale.models.integration_models.synqly_models.connectors import Edr, Ticketing, Vulnerabilities
-    from synqly.engine.resources.ocsf.resources.v_1_1_0.resources.securityfinding import ResourceDetails, Vulnerability
+    from synqly.engine.resources.ocsf.resources.v_1_3_0.resources.securityfinding import (
+        Finding,
+        ResourceDetails,
+        Vulnerability,
+    )
 
 from synqly import engine
 from synqly.engine import CreateTicketRequest
@@ -78,7 +82,7 @@ class Mapper:
         """
         due_date = convert_datetime_to_regscale_string(ticket.due_date)
         if not due_date:
-            due_date = self._determine_due_date(ticket.priority)
+            due_date = self._determine_due_date(ticket.priority, connector)
         ticket_dict = ticket.dict()
         key_val_desc = "All fields:\n"
         for k, v in ticket_dict.items():
@@ -102,9 +106,8 @@ class Mapper:
             regscale_issue.manualDetectionId = ticket.id
         return regscale_issue
 
-    @staticmethod
     def _ocsf_asset_to_regscale(
-        connector: Union["Edr", "Vulnerabilities"], asset: Union[InventoryInfo, OCSFAsset, SoftwareInfo], **kwargs
+        self, connector: Union["Edr", "Vulnerabilities"], asset: Union[InventoryInfo, OCSFAsset, SoftwareInfo], **kwargs
     ) -> IntegrationAsset:
         """
         Convert OCSF Asset to RegScale Asset
@@ -130,6 +133,9 @@ class Mapper:
         else:
             name = device_data.name or device_data.hostname or f"{connector.provider} Asset: {device_data.uid}"
             category = AssetCategory.Software
+        ip_v4s, ip_v6s = self._determine_ip_addresses(
+            device_data.ip_addresses if device_data.ip_addresses else [device_data.ip]
+        )
         return IntegrationAsset(
             name=name,
             identifier=device_data.uid,
@@ -139,7 +145,8 @@ class Mapper:
             parent_module=SecurityPlan.get_module_string(),
             mac_address=device_data.mac,
             fqdn=device_data.hostname,
-            ip_address=", ".join(device_data.ip_addresses) if device_data.ip_addresses else device_data.ip,
+            ip_address=", ".join(ip_v4s),
+            ipv6_address=", ".join(ip_v6s),
             location=device_data.location or device_data.zone,
             vlan_id=device_data.vlan_uid,
             other_tracking_number=device_data.uid,
@@ -152,7 +159,64 @@ class Mapper:
         )
 
     @staticmethod
-    def _parse_finding_data(finding: "SecurityFinding", vuln: Optional["Vulnerability"] = None) -> dict:
+    def _determine_ip_addresses(ips: list[str]) -> tuple[list[str], list[str]]:
+        """
+        Parse the list of ips and return a tuple of two lists: list of IP v4 and IP v6 addresses
+
+        :param list[str] ips: List of IP addresses
+        :return: Tuple containing two lists, list of IP v4 addresses and list of IP v6 addresses
+        :rtype: tuple[list[str], list[str]]
+        """
+        import ipaddress
+
+        ip_v4s = []
+        ip_v6s = []
+        for ip in ips:
+            try:
+                ipaddress.IPv4Address(ip)
+                ip_v4s.append(ip)
+            except ipaddress.AddressValueError:
+                try:
+                    ipaddress.IPv6Address(ip)
+                    ip_v6s.append(ip)
+                except ipaddress.AddressValueError:
+                    continue
+        return ip_v4s, ip_v6s
+
+    @staticmethod
+    def _determine_date(
+        attribute: str, finding: Optional["Finding"] = None, vuln: Optional["Vulnerability"] = None
+    ) -> datetime:
+        """
+        Determine the date based on the provided vulnerability or finding
+
+        :param str attribute: The attribute to determine the date for
+        :param Optional[SecurityFinding] finding: The finding to determine the date for
+        :param Optional[Vulnerability] vuln: The vulnerability to determine the date for
+        :return: The date
+        :rtype: datetime
+        """
+        from regscale.core.utils.date import normalize_timestamp
+
+        vuln_date = getattr(vuln, attribute) if vuln else None
+        fallback_vuln_date = getattr(vuln, attribute.replace("_dt", ""), None) if vuln else None
+        finding_date = getattr(finding, attribute) if finding else None
+        fallback_finding_date = getattr(finding, attribute.replace("_dt", ""), None) if finding else None
+        if vuln_date:
+            return vuln_date
+        elif finding_date:
+            return finding_date
+        # No datetime objects, lets try the epoch integers
+        if fallback_vuln_date:
+            fallback_vuln_date = normalize_timestamp(fallback_vuln_date)
+            return datetime.fromtimestamp(fallback_vuln_date)
+        elif fallback_finding_date:
+            fallback_finding_date = normalize_timestamp(fallback_finding_date)
+            return datetime.fromtimestamp(fallback_finding_date)
+        else:
+            return datetime.now()
+
+    def _parse_finding_data(self, finding: "SecurityFinding", vuln: Optional["Vulnerability"] = None) -> dict:
         """
         Parse the data from the SecurityFinding object
 
@@ -160,20 +224,25 @@ class Mapper:
         :return: A dictionary of the parsed data
         :rtype: dict
         """
-        from synqly.engine.resources.ocsf.resources.v_1_1_0.resources.securityfinding import Remediation
+        from synqly.engine.resources.ocsf.resources.v_1_3_0.resources.securityfinding import Remediation
 
         finding_data = {
             "cve": None,
-            "first_seen": vuln.first_seen_time_dt if vuln else finding.finding.first_seen_time_dt,
-            "last_seen": vuln.last_seen_time_dt if vuln else finding.finding.last_seen_time_dt,
+            "first_seen": self._determine_date("first_seen_time_dt", getattr(finding, "finding", None), vuln),
+            "last_seen": self._determine_date("last_seen_time_dt", getattr(finding, "finding", None), vuln),
             "plugin_id": vuln.cve.uid if vuln else finding.finding.product_uid,
-            "severity": int(vuln.severity) if vuln else finding.severity_id,
+            "severity": vuln.severity if vuln and getattr(vuln, "severity") else finding.severity_id,
             "remediation": getattr(vuln, "remediation") if vuln else finding.finding.remediation.desc,
+            "title": getattr(vuln, "title") or finding.finding.title,
         }
         if vuln:
             finding_data["cve"] = vuln.cve.uid
         else:
             finding_data["cve"] = finding.finding.title if "cve" in finding.finding.title.lower() else None
+        try:
+            finding_data["severity"] = int(finding_data["severity"])
+        except ValueError:
+            finding_data["severity"] = 0
         if isinstance(finding_data["remediation"], Remediation):
             finding_data["remediation"] = finding_data["remediation"].desc
         elif isinstance(finding_data["remediation"], dict):
@@ -183,8 +252,26 @@ class Mapper:
             finding_data["remediation"] = remediation_string
         elif isinstance(finding_data["remediation"], list):
             finding_data["remediation"] = "\n".join(finding_data["remediation"])
+        finding_data["title"] = self._determine_title(finding_data["title"], finding_data["cve"])
         return finding_data
 
+    @staticmethod
+    def _determine_title(title: Optional[str], cve: Optional[str]) -> str:
+        """
+        Determine the title based on the provided title and cve
+
+        :param Optional[str] title: The title to determine the title for, defaults to None
+        :param Optional[str] cve: The cve to determine the title for, defaults to None
+        :return: The title
+        :rtype: str
+        """
+        if title and cve:
+            return f"{title} - {cve}"
+        elif title is None and cve:
+            return cve
+        else:
+            return title
+
     @staticmethod
     def _populate_cvs_scores(finding_obj: IntegrationFinding, vuln: Optional["Vulnerability"] = None) -> None:
         """
@@ -203,6 +290,17 @@ class Mapper:
                 else:
                     finding_obj.cvss_score = cvs.base_score
 
+        # validate that only one of the CVSS scores is populated
+        if finding_obj.cvss_v3_score:
+            finding_obj.cvss_v2_score = None
+            finding_obj.cvss_score = None
+        elif finding_obj.cvss_v2_score:
+            finding_obj.cvss_v3_score = None
+            finding_obj.cvss_score = None
+        elif finding_obj.cvss_score:
+            finding_obj.cvss_v3_score = None
+            finding_obj.cvss_v2_score = None
+
     def _security_finding_to_regscale(
         self, connector: "Vulnerabilities", finding: SecurityFinding, **_
     ) -> list[IntegrationFinding]:
@@ -227,28 +325,29 @@ class Mapper:
             """
             base = vuln if vuln else finding.finding
             finding_data = self._parse_finding_data(finding, vuln)
-            if resource.data:
-                dns = resource.data.get("hostname")
-                ip_address = resource.data.get("ipv4")
-            else:
-                dns = resource.uid if vuln else None
-                ip_address = None
+            resource_data = getattr(resource, "data", {})
+            dns = resource_data.get("hostname") or resource.uid if vuln else None
+            ip_v4s, ip_v6s = self._determine_ip_addresses(
+                resource_data["ipAddresses"] if resource_data.get("ipAddresses") else [resource_data.get("ip")]
+            )
+            ips = ip_v4s + ip_v6s
 
             finding_obj = IntegrationFinding(
                 control_labels=[],
                 category=f"{connector.integration_name} Vulnerability",
-                title=base.title,
+                title=finding_data["title"],
                 plugin_name=connector.integration_name,
                 severity=Issue.assign_severity(finding.severity),  # type: ignore
                 description=base.desc,
                 status=finding.status or "Open",
                 first_seen=self._datetime_to_str(finding_data["first_seen"]),
                 last_seen=self._datetime_to_str(finding_data["last_seen"]),
-                ip_address=ip_address,
+                ip_address=", ".join(ips),
                 plugin_id=finding_data["plugin_id"],
+                due_date=self._determine_due_date(finding_data["severity"], connector),
                 dns=dns,
                 severity_int=finding_data["severity"],
-                issue_title=base.title,
+                issue_title=finding_data["title"],
                 cve=finding_data["cve"],
                 evidence=finding.evidence,
                 impact=finding.impact,
@@ -330,22 +429,22 @@ class Mapper:
         else:
             return Priority.LOW
 
-    def _determine_due_date(self, severity: str) -> str:
+    def _determine_due_date(self, severity: str, connector: Union["Ticketing", "Vulnerabilities"]) -> str:
         """
         Determine the due date based on the provided severity
 
         :param str severity: RegScale severity
+        :param Union["Ticketing", "Vulnerabilities"] connector: Ticketing or Vulnerabilities connector class object
         :return: Due date for the issue
         :rtype: str
         """
-        from datetime import timedelta
-
         if "high" in severity.lower():
-            return (datetime.now() + timedelta(days=30)).strftime(self.date_format)
+            default_days = 30
         elif "medium" in severity.lower():
-            return (datetime.now() + timedelta(days=60)).strftime(self.date_format)
+            default_days = 90
         else:
-            return (datetime.now() + timedelta(days=90)).strftime(self.date_format)
+            default_days = 180
+        return Issue.get_due_date(severity, connector.app.config, connector.integration, default_days=default_days)
 
     def _regscale_issue_to_ticket(self, regscale_issue: Issue, **kwargs) -> CreateTicketRequest:
         """

regscale/models/integration_models/synqly_models/synqly_model.py

@@ -20,6 +20,7 @@ from regscale.core.app.api import Api
 from regscale.core.app.application import Application
 from regscale.core.app.utils.app_utils import create_progress_object, error_and_exit
 from regscale.models.integration_models.synqly_models.connector_types import ConnectorType
+from regscale.models.integration_models.synqly_models.filter_parser import FilterParser
 from regscale.models.integration_models.synqly_models.ocsf_mapper import Mapper
 from regscale.models.integration_models.synqly_models.param import Param
 from regscale.models.integration_models.synqly_models.tenants import Tenant
@@ -37,7 +38,7 @@ class SynqlyModel(BaseModel, ABC):
     client: Optional[Any] = None
     connectors: dict = Field(default_factory=dict)
     # defined using the openApi spec on 7/16/2024, this is updated via _get_integrations_and_secrets()
-    connector_types: set = Field(default_factory=lambda: set([connector.__str__() for connector in ConnectorType]))
+    connector_types: set = Field(default_factory=lambda: {connector.__str__() for connector in ConnectorType})
     terminated: Optional[bool] = False
     app: Application = Field(default_factory=Application, alias="app")
     api: Api = Field(default_factory=Api, alias="api")
@@ -60,6 +61,7 @@ class SynqlyModel(BaseModel, ABC):
     created_regscale_objects: list = Field(default_factory=list)
     updated_regscale_objects: list = Field(default_factory=list)
     regscale_objects_to_update: list = Field(default_factory=list)
+    filter_parser: Optional[FilterParser] = None
 
     def __init__(self: S, connector_type: Optional[str] = None, integration: Optional[str] = None, **kwargs):
         try:
@@ -241,7 +243,14 @@ class SynqlyModel(BaseModel, ABC):
                 config[f"{self._connector_type}_{self.integration}_{key}"] = attribute[key].default
             elif not skip_prompts:
                 self.logger.info(f"Enter the {key} for {self.integration}. Description: {attribute[key].description}")
-                provided_secret = input(f"{key}: ")
+                if key.lower() in ["secret", "password"] or "token" in key.lower():
+                    from getpass import getpass
+
+                    print(f"{attribute[key].description} (input will be hidden)")
+                    provided_secret = getpass(f"{key}: ")
+                else:
+                    print(f"{attribute[key].description} (input will be visible)")
+                    provided_secret = input(f"{key}: ")
                 kwargs[key] = provided_secret
                 config[f"{self._connector_type}_{self.integration}_{key}"] = provided_secret
             else:
@@ -278,6 +287,8 @@ class SynqlyModel(BaseModel, ABC):
         :rtype: dict
         """
         raw_data = self._load_from_package()
+        # Initialize FilterParser with the loaded capabilities data
+        self.filter_parser = FilterParser(capabilities_data=raw_data)
         return self._parse_api_spec_data(raw_data, return_params)
 
     def _parse_api_spec_data(self, data: dict, return_params: bool = False) -> dict:
@@ -441,12 +452,12 @@ class SynqlyModel(BaseModel, ABC):
 
         operations = schema.get("operations", [])
         capabilities = [item["name"] for item in operations if item.get("supported")]
-        capabilities_params = list(
+        capabilities_params = [
             field
             for item in operations
             if item.get("supported") and "required_fields" in item.keys()
             for field in item.get("required_fields", [])
-        )
+        ]
         if self.integration.lower() in key.lower():
             self.capabilities = capabilities
         schema = schema["provider_config"]
@@ -636,6 +647,42 @@ class SynqlyModel(BaseModel, ABC):
         """
         pass
 
+    def validate_filters(self, filters: Union[tuple, list, str]) -> list[str]:
+        """
+        Validate filter strings against provider capabilities.
+
+        :param Union[tuple, list, str] filters: Filter(s) to validate
+        :return: Validated filter list
+        :rtype: list[str]
+        :raises: SystemExit if validation fails
+        """
+        if not self.filter_parser:
+            self.logger.warning("FilterParser not available for filter validation")
+            if isinstance(filters, list):
+                return filters
+            elif filters:
+                return [filters]
+            else:
+                return []
+
+        provider_id = f"{self._connector_type}_{self.integration}"
+        validated_filters = []
+
+        # Normalize to list for processing
+        if isinstance(filters, str):
+            filters = [filters]
+        elif filters is None:
+            return []
+
+        for filter_string in filters:
+            is_valid, error_message = self.filter_parser.validate_filter(provider_id, filter_string)
+            if not is_valid:
+                error_and_exit(f"Filter validation failed: {error_message}")
+            validated_filters.append(filter_string)
+            self.logger.debug(f"Filter '{filter_string}' validated successfully")
+
+        return validated_filters
+
     def fetch_integration_data(
         self, func: Callable, **kwargs
     ) -> list[Union["InventoryAsset", "SecurityFinding", "Ticket"]]:
@@ -648,11 +695,12 @@ class SynqlyModel(BaseModel, ABC):
         """
         query_filter = kwargs.get("filter")
         limit = kwargs.get("limit", 200)
+
+        # Validate filters if provided
+        if query_filter:
+            query_filter = self.validate_filters(query_filter)
         integration_data: list = []
-        fetch_res = func(
-            filter=query_filter,
-            limit=limit,
-        )
+        fetch_res = self._fetch_data_with_retries(func=func, limit=limit, filter=query_filter)
         self.logger.info(f"Received {len(fetch_res.result)} record(s) from {self.integration_name}.")
         integration_data.extend(fetch_res.result)
         # check and handle pagination
@@ -660,24 +708,49 @@ class SynqlyModel(BaseModel, ABC):
             try:
                 # fetch.cursor can be an int as a string, or a continuation token
                 while int(fetch_res.cursor) == len(integration_data):
-                    fetch_res = func(
-                        filter=query_filter,
-                        limit=limit,
-                        cursor=fetch_res.cursor,
+                    fetch_res = self._fetch_data_with_retries(
+                        func=func, limit=limit, cursor=fetch_res.cursor, filter=query_filter
                     )
                     integration_data.extend(fetch_res.result)
             except ValueError:
                 while fetch_res.cursor:
-                    fetch_res = func(
-                        filter=query_filter,
-                        limit=limit,
-                        cursor=fetch_res.cursor,
+                    fetch_res = self._fetch_data_with_retries(
+                        func=func, limit=limit, cursor=fetch_res.cursor, filter=query_filter
                     )
                     integration_data.extend(fetch_res.result)
                     self.logger.info(f"Received {len(integration_data)} record(s) from {self.integration_name}...")
         self.logger.info(f"Fetched {len(integration_data)} total record(s) from {self.integration_name}...")
         return integration_data
 
+    def _fetch_data_with_retries(
+        self, func: Callable, limit: int, cursor: Union[str, int, None] = None, filter: Optional[str] = None
+    ) -> Any:
+        """
+        Fetches data from the integration using the provided function and handles pagination
+
+        :param Callable func: The function to fetch data from the integration
+        :param int limit: The limit of records to fetch
+        :param Optional[str] filter: The filter to apply to the data
+        :return: The data from the integration
+        :rtype: Any
+        """
+        import time
+
+        retries = 0
+        sleep_timers = [10, 20, 40]
+        while retries < 3:
+            try:
+                if cursor:
+                    return func(limit=limit, filter=filter, cursor=cursor)
+                else:
+                    return func(limit=limit, filter=filter)
+            except Exception as e:
+                self.logger.error(f"Error fetching data with retries: {e}")
+                self.logger.error(f"Retrying in {sleep_timers[retries]} seconds...")
+                time.sleep(sleep_timers[retries])
+                retries += 1
+        error_and_exit("Failed to fetch data after 3 retries")
+
     def run_integration_sync(self, *args, **kwargs) -> None:
         """
         Runs the sync process for the integration

regscale/models/locking.py

@@ -90,11 +90,15 @@ class FileLock:
 
         :rtype: None
         """
-        if os.path.isfile(self.lock_file):
-            with open(self.lock_file, "r") as lockfile:
-                scope = str(lockfile.read().strip())
-
-            if scope == str(self.lock_scope):
-                os.remove(self.lock_file)  # Lock released
-            else:
-                pass  # Lock can only be released by {self.lock_scope}
+        try:
+            if os.path.isfile(self.lock_file):
+                with open(self.lock_file, "r") as lockfile:
+                    scope = str(lockfile.read().strip())
+
+                if scope == str(self.lock_scope):
+                    os.remove(self.lock_file)  # Lock released
+                else:
+                    pass  # Lock can only be released by {self.lock_scope}
+        except FileNotFoundError:
+            # Lock file was already removed by another process, this is acceptable in parallel execution
+            pass

regscale/models/platform.py

@@ -5,10 +5,9 @@ from typing import Optional, Union
 
 from pydantic import BaseModel, SecretStr
 
+from regscale.core.app.api import Api
 from regscale.core.login import get_regscale_token
 from regscale.core.utils.urls import generate_regscale_domain_url
-from regscale.core.app.api import Api
-from regscale.core.app.application import Application
 
 
 class RegScaleAuth(BaseModel):
@@ -45,6 +44,7 @@ class RegScaleAuth(BaseModel):
         password: Optional[Union[str, SecretStr]] = None,
         domain: Optional[str] = None,
         mfa_token: Optional[str] = None,
+        app_id: Optional[int] = 1,
     ) -> "RegScaleAuth":
         """
         Authenticate with RegScale and return a token and a user_id
@@ -53,6 +53,7 @@ class RegScaleAuth(BaseModel):
         :param Optional[Union[str, SecretStr]] password: Password to log in with, defaults to None
         :param Optional[str] domain: Domain to log into, defaults to None
         :param Optional[str] mfa_token: mfa_token to verify, defaults to None
+        :param Optional[int] app_id: The app ID to login with
         :return: RegScaleAuth object
         :rtype: RegScaleAuth
         """
@@ -89,6 +90,7 @@ class RegScaleAuth(BaseModel):
             password=password.get_secret_value(),
             domain=domain,
             mfa_token=mfa_token,
+            app_id=app_id,
         )
         return cls(
             user_id=uid,

regscale/models/regscale_models/__init__.py

@@ -41,6 +41,7 @@ from .implementation_objective import *
 from .implementation_option import *
 from .implementation_role import *
 from .incident import *
+from .inheritance import *
 from .inherited_control import *
 from .interconnection import *
 from .issue import *
@@ -50,7 +51,10 @@ from .link import *
 from .master_assessment import *
 from .milestone import *
 from .meta_data import *
+from .module import *
+from .modules import *
 from .objective import *
+from .organization import *
 from .parameter import *
 from .policy import *
 from .ports_protocol import *
@@ -79,10 +83,13 @@ from .stig import *
 from .supply_chain import *
 from .system_role import *
 from .system_role_external_assignment import *
+from .tag import *
+from .tag_mapping import *
 from .task import *
 from .threat import *
 from .team import *
 from .user import *
+from .user_group import *
 from .vulnerability import *
 from .vulnerability_mapping import *
 from .workflow import *

regscale/models/regscale_models/assessment.py

@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
 from enum import Enum
 from typing import Any, List, Optional, Union
 
+from pydantic import Field
 from requests import JSONDecodeError
 
 from regscale.core.app.api import Api
@@ -60,7 +61,7 @@ class Assessment(RegScaleModel):
     _module_slug = "assessments"
 
     id: int = 0
-    leadAssessorId: str = ""
+    leadAssessorId: str = Field(default_factory=RegScaleModel.get_user_id)
     title: Optional[str] = None
     assessmentType: Optional[Union[AssessmentType, str]] = None
     plannedStart: Optional[str] = None

regscale/models/regscale_models/catalog.py

@@ -273,5 +273,5 @@ class Catalog(RegScaleModel):
         response = cls._get_api_handler().get(endpoint)
 
         if response and response.ok and response.status_code not in [204, 404]:
-            return cls.model_validate(response.json())
+            return cls(**response.json())
         return None