regscale-cli 6.24.0.0__py3-none-any.whl → 6.25.0.0__py3-none-any.whl

regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py

@@ -99,11 +99,17 @@ class Vulnerabilities(SynqlyModel):
         """
         vuln_filter = self._handle_scan_date_options(regscale_ssp_id=regscale_ssp_id, **kwargs)
         self.logger.debug(f"Vulnerability filter: {vuln_filter}")
+
+        # Pop the filter from kwargs so it doesn't get passed to query_findings
+        asset_filter = kwargs.pop("filter", [])
+        if asset_filter:
+            self.logger.debug(f"Asset filter: {asset_filter}")
+
         self.logger.info(f"Fetching vulnerability data from {self.integration_name}...")
         findings = (
             self.fetch_integration_data(
                 func=self.tenant.engine_client.vulnerabilities.query_findings,
-                filter=vuln_filter,
+                filter=vuln_filter,  # Only severity/date filters for findings
                 **kwargs,
             )
             if self.can_fetch_vulns
@@ -111,7 +117,11 @@ class Vulnerabilities(SynqlyModel):
         )
         self.logger.info(f"Fetching asset data from {self.integration_name}...")
         assets = (
-            self.fetch_integration_data(func=self.tenant.engine_client.vulnerabilities.query_assets, **kwargs)
+            self.fetch_integration_data(
+                func=self.tenant.engine_client.vulnerabilities.query_assets,
+                filter=asset_filter,  # Field-based filters only for assets
+                **kwargs,
+            )
             if self.can_fetch_assets
             else []
         )

regscale/models/integration_models/synqly_models/filter_parser.py

@@ -0,0 +1,332 @@
+"""
+Centralized parser for extracting filter definitions from Synqly capabilities.
+Used by both code generation and query builder.
+"""
+
+import json
+import re
+from typing import Dict, List, Optional, Set, Tuple
+import importlib.resources as pkg_resources
+
+from regscale.models.integration_models.synqly_models.connector_types import ConnectorType
+
+
+class FilterParser:
+    """Parser for Synqly filter definitions from capabilities.json"""
+
+    # Define which connectors support filtering
+    FILTERABLE_CONNECTORS: Set[str] = {ConnectorType.Assets.value, ConnectorType.Vulnerabilities.value}
+
+    def __init__(self, capabilities_data: Optional[List[dict]] = None):
+        """
+        Initialize the filter parser with provided or loaded capabilities.
+
+        :param Optional[List[dict]] capabilities_data: Pre-loaded capabilities data.
+            If None, will load from package resources.
+        """
+        if capabilities_data is None:
+            self.capabilities_data = self._load_capabilities()
+        else:
+            self.capabilities_data = capabilities_data
+        self.filter_mapping = self._build_filter_mapping()
+
+    def _load_capabilities(self) -> List[dict]:
+        """
+        Load capabilities.json from package resources.
+
+        :return: List of capability definitions
+        :rtype: List[dict]
+        """
+        try:
+            files = pkg_resources.files("regscale.models.integration_models.synqly_models")
+            capabilities_file = files / "capabilities.json"
+            with capabilities_file.open("r") as file:
+                data = json.load(file)
+                return data.get("result", [])
+        except Exception as e:
+            print(f"Error loading capabilities.json: {e}")
+            return []
+
+    def _build_filter_mapping(self) -> Dict[str, Dict[str, List[dict]]]:
+        """
+        Build comprehensive filter mapping for all providers.
+
+        Structure:
+        {
+            'assets_armis_centrix': {
+                'query_devices': [
+                    {
+                        'name': 'device.ip',
+                        'type': 'string',
+                        'operators': ['eq', 'ne', 'in', 'not_in'],
+                        'values': []  # For enum types
+                    },
+                    ...
+                ]
+            },
+            'vulnerabilities_qualys': {
+                'query_findings': [...],
+                'query_assets': [...]
+            },
+            ...
+        }
+
+        :return: Mapping of provider IDs to their operations and filters
+        :rtype: Dict[str, Dict[str, List[dict]]]
+        """
+        filter_mapping = {}
+
+        for provider in self.capabilities_data:
+            provider_id = provider.get("id", "")
+            connector_type = provider.get("connector", "")
+
+            # Only process filterable connector types
+            if connector_type not in self.FILTERABLE_CONNECTORS:
+                continue
+
+            # Skip mock providers if desired (they end with _mock)
+            # if provider_id.endswith('_mock'):
+            #     continue
+
+            operations = provider.get("operations", [])
+            for operation in operations:
+                # Only process supported operations with filters
+                if not operation.get("supported", False):
+                    continue
+
+                filters = operation.get("filters", [])
+                if filters:
+                    if provider_id not in filter_mapping:
+                        filter_mapping[provider_id] = {}
+
+                    operation_name = operation.get("name", "")
+                    filter_mapping[provider_id][operation_name] = filters
+
+        return filter_mapping
+
+    def get_filters_for_provider(self, provider_id: str, operation: Optional[str] = None) -> List[dict]:
+        """
+        Get filters for a specific provider and optionally a specific operation.
+
+        :param str provider_id: Provider ID (e.g., 'assets_armis_centrix')
+        :param Optional[str] operation: Operation name (e.g., 'query_devices')
+        :return: List of filter definitions
+        :rtype: List[dict]
+        """
+        provider_filters = self.filter_mapping.get(provider_id, {})
+
+        if operation:
+            return provider_filters.get(operation, [])
+
+        # Return all filters for all operations if no specific operation
+        all_filters = []
+        seen_fields = set()  # Avoid duplicates
+
+        for op_filters in provider_filters.values():
+            for filter_def in op_filters:
+                field_name = filter_def.get("name", "")
+                if field_name not in seen_fields:
+                    all_filters.append(filter_def)
+                    seen_fields.add(field_name)
+
+        return all_filters
+
+    def get_providers_with_filters(self, connector_type: str) -> List[str]:
+        """
+        Get list of providers that support filtering for a connector type.
+
+        :param str connector_type: Connector type (e.g., 'assets', 'vulnerabilities')
+        :return: List of provider IDs that have filters
+        :rtype: List[str]
+        """
+        providers = []
+
+        for provider_id, operations in self.filter_mapping.items():
+            # Check if provider matches connector type and has filters
+            if provider_id.startswith(f"{connector_type}_") and operations:
+                providers.append(provider_id)
+
+        # Sort for consistent ordering
+        return sorted(providers)
+
+    def has_filters(self, provider_id: str) -> bool:
+        """
+        Check if a provider has any filters defined.
+
+        :param str provider_id: Provider ID to check
+        :return: True if provider has filters
+        :rtype: bool
+        """
+        return provider_id in self.filter_mapping and bool(self.filter_mapping[provider_id])
+
+    @staticmethod
+    def format_filter_string(field: str, operator: str, value: str) -> str:
+        """
+        Convert user input to Synqly filter format.
+
+        :param str field: Field name (e.g., 'device.ip')
+        :param str operator: Operator (e.g., 'eq', 'gte')
+        :param str value: Filter value
+        :return: Formatted filter string
+        :rtype: str
+
+        Example:
+            format_filter_string('device.ip', 'eq', '192.168.1.1')
+            Returns: 'device.ip[eq]192.168.1.1'
+        """
+        return f"{field}[{operator}]{value}"
+
+    @staticmethod
+    def parse_filter_string(filter_string: str) -> Optional[Tuple[str, str, str]]:
+        """
+        Parse a filter string into its components.
+
+        :param str filter_string: Filter in format 'field[operator]value'
+        :return: Tuple of (field, operator, value) or None if invalid
+        :rtype: Optional[Tuple[str, str, str]]
+        """
+        match = re.match(r"^([a-z._]+)\[([a-z_]+)\](.+)$", filter_string, re.IGNORECASE)
+        if match:
+            return match.groups()
+        return None
+
+    def validate_filter(self, provider_id: str, filter_string: str) -> Tuple[bool, str]:
+        """
+        Validate a filter string against provider capabilities.
+
+        :param str provider_id: Provider ID (e.g., 'assets_armis_centrix')
+        :param str filter_string: Filter in format 'field[operator]value'
+        :return: Tuple of (is_valid, error_message)
+        :rtype: Tuple[bool, str]
+        """
+        # Parse the filter string
+        parsed = self.parse_filter_string(filter_string)
+        if not parsed:
+            return False, f"Invalid filter format: {filter_string}. Expected format: field[operator]value"
+
+        field, operator, value = parsed
+
+        # Get all filters for this provider
+        provider_filters = self.get_filters_for_provider(provider_id)
+
+        if not provider_filters:
+            return False, f"Provider '{provider_id}' does not support filtering"
+
+        # Check if field exists
+        field_filter = None
+        for f in provider_filters:
+            if f.get("name") == field:
+                field_filter = f
+                break
+
+        if not field_filter:
+            available_fields = [f.get("name", "") for f in provider_filters]
+            return (
+                False,
+                f"Field '{field}' not supported by {provider_id}. Available fields: {', '.join(available_fields)}",
+            )
+
+        # Check if operator is valid for this field
+        valid_operators = field_filter.get("operators", [])
+        if operator not in valid_operators:
+            return (
+                False,
+                f"Operator '{operator}' not valid for field '{field}'. Valid operators: {', '.join(valid_operators)}",
+            )
+
+        # Optionally validate value type and format
+        field_type = field_filter.get("type", "string")
+
+        # Handle comma-separated values for 'in' and 'not_in' operators
+        if operator in ["in", "not_in"]:
+            values_to_check = [v.strip() for v in value.split(",")]
+        else:
+            values_to_check = [value]
+
+        for val in values_to_check:
+            if field_type == "number":
+                try:
+                    float(val)
+                except ValueError:
+                    return False, f"Value '{val}' is not a valid number for field '{field}'"
+            elif field_type == "enum":
+                valid_values = field_filter.get("values", [])
+                if valid_values and val not in valid_values:
+                    return (
+                        False,
+                        f"Value '{val}' not valid for field '{field}'. Valid values: {', '.join(valid_values)}",
+                    )
+
+        return True, ""
+
+    def get_operator_display_name(self, operator: str) -> str:
+        """
+        Get human-friendly display name for an operator.
+
+        :param str operator: Operator code
+        :return: Display name
+        :rtype: str
+        """
+        operator_map = {
+            "eq": "equals",
+            "ne": "not equals",
+            "in": "in list",
+            "not_in": "not in list",
+            "like": "matches pattern",
+            "not_like": "does not match pattern",
+            "gt": "greater than",
+            "gte": "greater than or equal to",
+            "lt": "less than",
+            "lte": "less than or equal to",
+        }
+        return operator_map.get(operator, operator)
+
+    def get_field_display_name(self, field: str) -> str:
+        """
+        Convert field name to human-friendly display name.
+
+        :param str field: Field name (e.g., 'device.hw_info.serial_number')
+        :return: Display name (e.g., 'Device Hardware Info Serial Number')
+        :rtype: str
+        """
+        # Replace dots and underscores with spaces, then title case
+        display = field.replace(".", " ").replace("_", " ").title()
+        return display
+
+    def get_connector_operations(self, connector_type: str) -> Dict[str, List[str]]:
+        """
+        Get all operations that support filtering for a connector type.
+
+        :param str connector_type: Connector type (e.g., 'assets')
+        :return: Dict mapping provider IDs to their filterable operations
+        :rtype: Dict[str, List[str]]
+        """
+        operations_map = {}
+
+        for provider_id, operations in self.filter_mapping.items():
+            if provider_id.startswith(f"{connector_type}_"):
+                operations_map[provider_id] = list(operations.keys())
+
+        return operations_map
+
+    def get_stats(self) -> dict:
+        """
+        Get statistics about loaded filters.
+
+        :return: Dictionary with filter statistics
+        :rtype: dict
+        """
+        stats = {
+            "total_providers": len(self.capabilities_data),
+            "providers_with_filters": len(self.filter_mapping),
+            "total_filters": 0,
+            "by_connector": {},
+        }
+
+        for connector in self.FILTERABLE_CONNECTORS:
+            providers = self.get_providers_with_filters(connector)
+            filter_count = sum(len(self.get_filters_for_provider(p)) for p in providers)
+            stats["by_connector"][connector] = {"providers": len(providers), "filters": filter_count}
+            stats["total_filters"] += filter_count
+
+        return stats

regscale/models/integration_models/synqly_models/synqly_model.py

@@ -20,6 +20,7 @@ from regscale.core.app.api import Api
 from regscale.core.app.application import Application
 from regscale.core.app.utils.app_utils import create_progress_object, error_and_exit
 from regscale.models.integration_models.synqly_models.connector_types import ConnectorType
+from regscale.models.integration_models.synqly_models.filter_parser import FilterParser
 from regscale.models.integration_models.synqly_models.ocsf_mapper import Mapper
 from regscale.models.integration_models.synqly_models.param import Param
 from regscale.models.integration_models.synqly_models.tenants import Tenant
@@ -37,7 +38,7 @@ class SynqlyModel(BaseModel, ABC):
     client: Optional[Any] = None
     connectors: dict = Field(default_factory=dict)
     # defined using the openApi spec on 7/16/2024, this is updated via _get_integrations_and_secrets()
-    connector_types: set = Field(default_factory=lambda: set([connector.__str__() for connector in ConnectorType]))
+    connector_types: set = Field(default_factory=lambda: {connector.__str__() for connector in ConnectorType})
     terminated: Optional[bool] = False
     app: Application = Field(default_factory=Application, alias="app")
     api: Api = Field(default_factory=Api, alias="api")
@@ -60,6 +61,7 @@ class SynqlyModel(BaseModel, ABC):
     created_regscale_objects: list = Field(default_factory=list)
     updated_regscale_objects: list = Field(default_factory=list)
     regscale_objects_to_update: list = Field(default_factory=list)
+    filter_parser: Optional[FilterParser] = None
 
     def __init__(self: S, connector_type: Optional[str] = None, integration: Optional[str] = None, **kwargs):
         try:
@@ -278,6 +280,8 @@ class SynqlyModel(BaseModel, ABC):
         :rtype: dict
         """
         raw_data = self._load_from_package()
+        # Initialize FilterParser with the loaded capabilities data
+        self.filter_parser = FilterParser(capabilities_data=raw_data)
         return self._parse_api_spec_data(raw_data, return_params)
 
     def _parse_api_spec_data(self, data: dict, return_params: bool = False) -> dict:
@@ -441,12 +445,12 @@ class SynqlyModel(BaseModel, ABC):
 
         operations = schema.get("operations", [])
         capabilities = [item["name"] for item in operations if item.get("supported")]
-        capabilities_params = list(
+        capabilities_params = [
             field
             for item in operations
             if item.get("supported") and "required_fields" in item.keys()
             for field in item.get("required_fields", [])
-        )
+        ]
         if self.integration.lower() in key.lower():
             self.capabilities = capabilities
         schema = schema["provider_config"]
@@ -636,6 +640,42 @@ class SynqlyModel(BaseModel, ABC):
         """
         pass
 
+    def validate_filters(self, filters: Union[tuple, list, str]) -> list[str]:
+        """
+        Validate filter strings against provider capabilities.
+
+        :param Union[tuple, list, str] filters: Filter(s) to validate
+        :return: Validated filter list
+        :rtype: list[str]
+        :raises: SystemExit if validation fails
+        """
+        if not self.filter_parser:
+            self.logger.warning("FilterParser not available for filter validation")
+            if isinstance(filters, list):
+                return filters
+            elif filters:
+                return [filters]
+            else:
+                return []
+
+        provider_id = f"{self._connector_type}_{self.integration}"
+        validated_filters = []
+
+        # Normalize to list for processing
+        if isinstance(filters, str):
+            filters = [filters]
+        elif filters is None:
+            return []
+
+        for filter_string in filters:
+            is_valid, error_message = self.filter_parser.validate_filter(provider_id, filter_string)
+            if not is_valid:
+                error_and_exit(f"Filter validation failed: {error_message}")
+            validated_filters.append(filter_string)
+            self.logger.debug(f"Filter '{filter_string}' validated successfully")
+
+        return validated_filters
+
     def fetch_integration_data(
         self, func: Callable, **kwargs
     ) -> list[Union["InventoryAsset", "SecurityFinding", "Ticket"]]:
@@ -648,6 +688,10 @@ class SynqlyModel(BaseModel, ABC):
         """
         query_filter = kwargs.get("filter")
         limit = kwargs.get("limit", 200)
+
+        # Validate filters if provided
+        if query_filter:
+            query_filter = self.validate_filters(query_filter)
         integration_data: list = []
         fetch_res = func(
             filter=query_filter,

regscale/models/regscale_models/compliance_settings.py

@@ -110,3 +110,31 @@ class ComplianceSettings(RegScaleModel):
         :rtype: List[str]
         """
         return self.__class__.get_labels(self.id, setting_field)
+
+    @classmethod
+    def get_settings_list(cls) -> List[dict]:
+        """
+        Get all compliance settings list items from settingsList endpoint.
+
+        :return: A list of compliance settings list items
+        :rtype: List[dict]
+        """
+        response = cls._get_api_handler().get(endpoint="/api/compliance/settingsList")
+        if response and response.ok:
+            return response.json()
+        return []
+
+    @classmethod
+    def get_default_responsibility_for_compliance_setting(cls, compliance_setting_id: int) -> Optional[str]:
+        """
+        Get the default responsibility value for a specific compliance setting.
+
+        :param int compliance_setting_id: The compliance setting ID
+        :return: The default responsibility value or None if not found
+        :rtype: Optional[str]
+        """
+        settings_list = cls.get_settings_list()
+        for setting in settings_list:
+            if setting.get("complianceSettingId") == compliance_setting_id and setting.get("isDefault", False):
+                return setting.get("statusName")
+        return None

regscale/models/regscale_models/component.py

@@ -76,6 +76,7 @@ class Component(RegScaleModel):
     externalId: Optional[str] = None
     isPublic: bool = True
     riskCategorization: Optional[str] = None
+    complianceSettingsId: Optional[int] = None
 
     @staticmethod
     def _get_additional_endpoints() -> ConfigDict: