regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

regscale/integrations/commercial/wizv2/core/__init__.py

@@ -0,0 +1,133 @@
+"""Core Wiz integration modules including authentication, client, file operations, and constants."""
+
+from regscale.integrations.commercial.wizv2.core.auth import (
+    AUTH0_URLS,
+    COGNITO_URLS,
+    generate_authentication_params,
+    get_token,
+    wiz_authenticate,
+)
+from regscale.integrations.commercial.wizv2.core.client import (
+    AsyncWizGraphQLClient,
+    run_async_queries,
+)
+from regscale.integrations.commercial.wizv2.core.file_operations import FileOperations
+from regscale.integrations.commercial.wizv2.core.constants import (
+    ASSET_TYPE_MAPPING,
+    BEARER,
+    CHECK_INTERVAL_FOR_DOWNLOAD_REPORT,
+    CLOUD_CONFIG_FINDING_QUERY,
+    CLOUD_CONFIG_FINDINGS_FILE_PATH,
+    CONTENT_TYPE,
+    CPE_PART_TO_CATEGORY_MAPPING,
+    CREATE_REPORT_QUERY,
+    DATA_FINDING_QUERY,
+    DATA_FINDINGS_FILE_PATH,
+    DATASOURCE,
+    DEFAULT_WIZ_HARDWARE_TYPES,
+    DOWNLOAD_QUERY,
+    END_OF_LIFE_FILE_PATH,
+    END_OF_LIFE_QUERY,
+    EXCESSIVE_ACCESS_FILE_PATH,
+    EXCESSIVE_ACCESS_QUERY,
+    EXTERNAL_ATTACK_SURFACE_FILE_PATH,
+    EXTERNAL_ATTACK_SURFACE_QUERY,
+    FRAMEWORK_CATEGORIES,
+    FRAMEWORK_MAPPINGS,
+    FRAMEWORK_SHORTCUTS,
+    HOST_VULNERABILITY_FILE_PATH,
+    HOST_VULNERABILITY_QUERY,
+    INVENTORY_FILE_PATH,
+    INVENTORY_QUERY,
+    ISSUE_QUERY,
+    ISSUES_FILE_PATH,
+    MAX_RETRIES,
+    NETWORK_EXPOSURE_FILE_PATH,
+    NETWORK_EXPOSURE_QUERY,
+    PROVIDER,
+    RATE_LIMIT_MSG,
+    RECOMMENDED_WIZ_INVENTORY_TYPES,
+    REPORTS_QUERY,
+    RERUN_REPORT_QUERY,
+    RESOURCE,
+    SBOM_FILE_PATH,
+    SBOM_QUERY,
+    SECRET_FINDINGS_FILE_PATH,
+    SECRET_FINDINGS_QUERY,
+    SEVERITY_MAP,
+    TECHNOLOGIES_FILE_PATH,
+    VULNERABILITY_FILE_PATH,
+    VULNERABILITY_QUERY,
+    WIZ_FRAMEWORK_QUERY,
+    WIZ_POLICY_QUERY,
+    WizVulnerabilityType,
+    get_compliance_report_variables,
+    get_wiz_issue_queries,
+    get_wiz_vulnerability_queries,
+)
+
+__all__ = [
+    # Auth
+    "AUTH0_URLS",
+    "COGNITO_URLS",
+    "generate_authentication_params",
+    "get_token",
+    "wiz_authenticate",
+    # Client
+    "AsyncWizGraphQLClient",
+    "run_async_queries",
+    # File Operations
+    "FileOperations",
+    # Constants
+    "ASSET_TYPE_MAPPING",
+    "BEARER",
+    "CHECK_INTERVAL_FOR_DOWNLOAD_REPORT",
+    "CLOUD_CONFIG_FINDING_QUERY",
+    "CLOUD_CONFIG_FINDINGS_FILE_PATH",
+    "CONTENT_TYPE",
+    "CPE_PART_TO_CATEGORY_MAPPING",
+    "CREATE_REPORT_QUERY",
+    "DATA_FINDING_QUERY",
+    "DATA_FINDINGS_FILE_PATH",
+    "DATASOURCE",
+    "DEFAULT_WIZ_HARDWARE_TYPES",
+    "DOWNLOAD_QUERY",
+    "END_OF_LIFE_FILE_PATH",
+    "END_OF_LIFE_QUERY",
+    "EXCESSIVE_ACCESS_FILE_PATH",
+    "EXCESSIVE_ACCESS_QUERY",
+    "EXTERNAL_ATTACK_SURFACE_FILE_PATH",
+    "EXTERNAL_ATTACK_SURFACE_QUERY",
+    "FRAMEWORK_CATEGORIES",
+    "FRAMEWORK_MAPPINGS",
+    "FRAMEWORK_SHORTCUTS",
+    "HOST_VULNERABILITY_FILE_PATH",
+    "HOST_VULNERABILITY_QUERY",
+    "INVENTORY_FILE_PATH",
+    "INVENTORY_QUERY",
+    "ISSUE_QUERY",
+    "ISSUES_FILE_PATH",
+    "MAX_RETRIES",
+    "NETWORK_EXPOSURE_FILE_PATH",
+    "NETWORK_EXPOSURE_QUERY",
+    "PROVIDER",
+    "RATE_LIMIT_MSG",
+    "RECOMMENDED_WIZ_INVENTORY_TYPES",
+    "REPORTS_QUERY",
+    "RERUN_REPORT_QUERY",
+    "RESOURCE",
+    "SBOM_FILE_PATH",
+    "SBOM_QUERY",
+    "SECRET_FINDINGS_FILE_PATH",
+    "SECRET_FINDINGS_QUERY",
+    "SEVERITY_MAP",
+    "TECHNOLOGIES_FILE_PATH",
+    "VULNERABILITY_FILE_PATH",
+    "VULNERABILITY_QUERY",
+    "WIZ_FRAMEWORK_QUERY",
+    "WIZ_POLICY_QUERY",
+    "WizVulnerabilityType",
+    "get_compliance_report_variables",
+    "get_wiz_issue_queries",
+    "get_wiz_vulnerability_queries",
+]

regscale/integrations/commercial/wizv2/{async_client.py → core/client.py}

@@ -4,7 +4,7 @@
 
 import asyncio
 import logging
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import anyio
 import httpx
@@ -48,7 +48,7 @@ class AsyncWizGraphQLClient:
         self,
         query: str,
         variables: Optional[Dict[str, Any]] = None,
-        progress_callback: Optional[callable] = None,
+        progress_callback: Optional[Callable] = None,
         task_name: str = "GraphQL Query",
     ) -> Dict[str, Any]:
         """
@@ -118,7 +118,7 @@ class AsyncWizGraphQLClient:
         query: str,
         variables: Dict[str, Any],
         topic_key: str,
-        progress_callback: Optional[callable] = None,
+        progress_callback: Optional[Callable] = None,
         task_name: str = "Paginated Query",
     ) -> List[Dict[str, Any]]:
         """
@@ -156,6 +156,10 @@ class AsyncWizGraphQLClient:
                 nodes = topic_data.get("nodes", [])
                 page_info = topic_data.get("pageInfo", {})
 
+                # Handle case where nodes is explicitly None
+                if nodes is None:
+                    nodes = []
+
                 all_nodes.extend(nodes)
 
                 has_next_page = page_info.get("hasNextPage", False)

regscale/integrations/commercial/wizv2/{constants.py → core/constants.py}

@@ -545,6 +545,76 @@ CREATE_REPORT_QUERY = """
     }
     }
 """
+
+
+def get_compliance_report_variables(
+    project_id: str, run_starts_at: Optional[str] = None, framework_ids: Optional[List[str]] = None
+) -> dict:
+    """Get compliance report variables with dynamic projectId and runStartsAt.
+
+    :param str project_id: The Wiz project ID
+    :param Optional[str] run_starts_at: ISO timestamp for when the report should start, defaults to current time
+    :param Optional[List[str]] framework_ids: List of framework IDs to include, defaults to NIST SP 800-53 Rev 5
+    :return: Variables for compliance report creation
+    :rtype: dict
+    """
+    from datetime import datetime, timezone
+
+    if not run_starts_at:
+        run_starts_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+    if not framework_ids:
+        # Default to NIST SP 800-53 Revision 5
+        framework_ids = ["wf-id-4"]
+
+    return {
+        "input": {
+            "name": f"Compliance Report - {project_id}",
+            "type": "COMPLIANCE_ASSESSMENTS",
+            "compressionMethod": "GZIP",
+            "runIntervalHours": 168,
+            "runStartsAt": run_starts_at,
+            "csvDelimiter": "US",
+            "projectId": project_id,
+            "complianceAssessmentsParams": {
+                "securityFrameworkIds": framework_ids,
+            },
+            "emailTargetParams": None,
+            "exportDestinations": None,
+            "columnSelection": [
+                "Assessed At",
+                "Category",
+                "Cloud Provider",
+                "Cloud Provider ID",
+                "Compliance Check Name (Wiz Subcategory)",
+                "Created At",
+                "Framework",
+                "Ignore Reason",
+                "Issue/Finding ID",
+                "Native Type",
+                "Object Type",
+                "Policy Description",
+                "Policy ID",
+                "Policy Name",
+                "Policy Short Name",
+                "Policy Type",
+                "Projects",
+                "Remediation Steps",
+                "Resource Cloud Platform",
+                "Resource Group Name",
+                "Resource ID",
+                "Resource Name",
+                "Resource Region",
+                "Result",
+                "Severity",
+                "Subscription",
+                "Subscription Name",
+                "Subscription Provider ID",
+            ],
+        }
+    }
+
+
 REPORTS_QUERY = """
         query ReportsTable($filterBy: ReportFilters, $first: Int, $after: String) {
           reports(first: $first, after: $after, filterBy: $filterBy) {
@@ -562,22 +632,6 @@ REPORTS_QUERY = """
               emailTarget {
                 to
               }
-              parameters {
-                query
-                framework {
-                  name
-                }
-                subscriptions {
-                  id
-                  name
-                  type
-                }
-                entities {
-                  id
-                  name
-                  type
-                }
-              }
               lastRun {
                 ...LastRunDetails
               }
@@ -1101,85 +1155,24 @@ fragment SecuritySubCategoryDetails on SecuritySubCategory {
 }
 """
 DATA_FINDING_QUERY = """
-query DataFindingsGroupedByValueTable($groupBy: DataFindingsGroupedByValueField!, $after: String, $first: Int, $filterBy: DataFindingFilters, $orderBy: DataFindingsGroupedByValueOrder) {
-  dataFindingsGroupedByValue(
-    groupBy: $groupBy
+query DataFindingsTable($after: String, $first: Int, $filterBy: DataFindingFiltersV2, $orderBy: DataFindingOrder, $fetchTotalCount: Boolean = true) {
+  dataFindingsV2(
     filterBy: $filterBy
     first: $first
     after: $after
     orderBy: $orderBy
   ) {
     nodes {
-      categories
-      location {
-        countryCode
-        state
-      }
-      regionCount
-      graphEntityCount
-      graphEntity {
-        id
-        name
-        type
-        properties
-        projects {
-          id
-          name
-          slug
-          isFolder
-        }
-        issues(filterBy: {status: [OPEN, IN_PROGRESS]}) {
-          criticalSeverityCount
-          highSeverityCount
-          mediumSeverityCount
-          lowSeverityCount
-          informationalSeverityCount
-        }
-      }
-      cloudAccount {
-        id
-        name
-        externalId
-        cloudProvider
-      }
-      dataClassifiers {
-        id
-        name
-        category
-        matcherType
-        severity
-      }
-      securitySubCategories {
-        id
-        title
-        externalId
-        description
-        category {
-          id
-          name
-          description
-          framework {
-            id
-            name
-            description
-            enabled
-          }
-        }
-      }
-      findingsCount
-      dataFindings(first: 5) {
-        nodes {
-          ...DataFindingDetails
-        }
-      }
+      ...DataFindingDetails
     }
     pageInfo {
       hasNextPage
       endCursor
     }
-    totalCount
+    totalCount @include(if: $fetchTotalCount)
   }
 }
+
 fragment DataFindingDetails on DataFinding {
   id
   name
@@ -1187,10 +1180,10 @@ fragment DataFindingDetails on DataFinding {
     id
     name
     category
+    isTenantSpecific
     securitySubCategories {
       id
       title
-      externalId
       description
       category {
         id
@@ -1216,8 +1209,12 @@ fragment DataFindingDetails on DataFinding {
     state
   }
   severity
+  status
   totalMatchCount
   uniqueMatchCount
+  maxUniqueMatchesReached
+  uniqueLocationsCount
+  isEntityPublic
   graphEntity {
     id
     name
@@ -1231,6 +1228,12 @@ fragment DataFindingDetails on DataFinding {
     }
   }
   externalSource
+  details {
+    applicationServices {
+      id
+      displayName
+    }
+  }
 }
 """
 
@@ -1295,8 +1298,8 @@ def get_wiz_vulnerability_queries(project_id: str, filter_by: Optional[dict] = N
                 "first": 200,
                 "quick": True,
                 "filterBy": {
-                    "rule": {},
-                    "resource": {"projectId": [project_id]},
+                    "resource": {"projectId": project_id},
+                    "status": ["OPEN", "IN_PROGRESS"],
                 },
             },
         },
@@ -1317,14 +1320,14 @@ def get_wiz_vulnerability_queries(project_id: str, filter_by: Optional[dict] = N
         {
             "type": WizVulnerabilityType.DATA_FINDING,
             "query": DATA_FINDING_QUERY,
-            "topic_key": "dataFindingsGroupedByValue",
+            "topic_key": "dataFindingsV2",
             "file_path": DATA_FINDINGS_FILE_PATH,
-            "asset_lookup": "resource",
+            "asset_lookup": "graphEntity",
             "variables": {
                 "first": 200,
+                "fetchTotalCount": True,
                 "filterBy": {"projectId": [project_id]},
-                "orderBy": {"field": "FINDING_COUNT", "direction": "DESC"},
-                "groupBy": "GRAPH_ENTITY",
+                "orderBy": {"field": "TOTAL_MATCHES", "direction": "DESC"},
             },
         },
         {

regscale/integrations/commercial/wizv2/core/file_operations.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""File operations module for Wiz integration - handles caching and file I/O."""
+
+import datetime
+import json
+import logging
+import os
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from regscale.core.app.utils.app_utils import check_file_path
+
+logger = logging.getLogger("regscale")
+
+
+class FileOperations:
+    """Handles file operations for Wiz integration including caching and data persistence."""
+
+    @staticmethod
+    def load_json_file(file_path: str) -> Optional[Any]:
+        """
+        Load data from a JSON file.
+
+        :param str file_path: Path to JSON file
+        :return: Loaded data or None if file doesn't exist or is invalid
+        :rtype: Optional[Any]
+        """
+        if not os.path.exists(file_path):
+            logger.debug(f"File does not exist: {file_path}")
+            return None
+
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                return json.load(f)
+        except (OSError, json.JSONDecodeError) as e:
+            logger.error(f"Error reading JSON file {file_path}: {e}")
+            return None
+
+    @staticmethod
+    def save_json_file(data: Any, file_path: str, create_dir: bool = True) -> bool:
+        """
+        Save data to a JSON file.
+
+        :param Any data: Data to save
+        :param str file_path: Path to save file
+        :param bool create_dir: Whether to create parent directory if needed
+        :return: True if successful, False otherwise
+        :rtype: bool
+        """
+        try:
+            if create_dir:
+                check_file_path(os.path.dirname(file_path))
+
+            with open(file_path, "w", encoding="utf-8") as f:
+                json.dump(data, f)
+
+            logger.debug(f"Saved data to {file_path}")
+            return True
+
+        except Exception as e:
+            logger.warning(f"Failed to save data to {file_path}: {e}")
+            return False
+
+    @staticmethod
+    def get_file_age(file_path: str) -> Optional[datetime.timedelta]:
+        """
+        Get the age of a file as a timedelta.
+
+        :param str file_path: Path to file
+        :return: File age or None if file doesn't exist
+        :rtype: Optional[datetime.timedelta]
+        """
+        if not os.path.exists(file_path):
+            return None
+
+        try:
+            file_mod_time = datetime.datetime.fromtimestamp(os.path.getmtime(file_path))
+            current_time = datetime.datetime.now()
+            return current_time - file_mod_time
+        except OSError as e:
+            logger.warning(f"Error getting file age for {file_path}: {e}")
+            return None
+
+    @staticmethod
+    def is_cache_valid(file_path: str, max_age_hours: float = 8) -> bool:
+        """
+        Check if a cache file is valid (exists and not too old).
+
+        :param str file_path: Path to cache file
+        :param float max_age_hours: Maximum age in hours before cache is invalid
+        :return: True if cache is valid, False otherwise
+        :rtype: bool
+        """
+        file_age = FileOperations.get_file_age(file_path)
+        if file_age is None:
+            return False
+
+        max_age = datetime.timedelta(hours=max_age_hours)
+        return file_age < max_age
+
+    @staticmethod
+    def load_cache_or_fetch(
+        file_path: str,
+        fetch_fn: Callable[[], Any],
+        max_age_hours: float = 8,
+        save_cache: bool = True,
+    ) -> Any:
+        """
+        Load data from cache if valid, otherwise fetch and optionally cache.
+
+        :param str file_path: Path to cache file
+        :param Callable fetch_fn: Function to call to fetch fresh data
+        :param float max_age_hours: Maximum cache age in hours
+        :param bool save_cache: Whether to save fetched data to cache
+        :return: Data from cache or freshly fetched
+        :rtype: Any
+        """
+        # Try to load from cache if valid
+        if FileOperations.is_cache_valid(file_path, max_age_hours):
+            logger.info(f"Using cached data from {file_path} (newer than {max_age_hours} hours)")
+            cached_data = FileOperations.load_json_file(file_path)
+            if cached_data is not None:
+                return cached_data
+
+        # Cache invalid or doesn't exist - fetch fresh data
+        file_age = FileOperations.get_file_age(file_path)
+        if file_age:
+            logger.info(
+                f"Cache file {file_path} is {file_age.total_seconds() / 3600:.1f} hours old - fetching new data"
+            )
+        else:
+            logger.info(f"Cache file {file_path} does not exist - fetching new data")
+
+        data = fetch_fn()
+
+        # Save to cache if requested
+        if save_cache:
+            FileOperations.save_json_file(data, file_path)
+
+        return data
+
+    @staticmethod
+    def search_json_files(
+        identifier: str,
+        file_paths: List[str],
+        match_fn: Callable[[Dict, str], bool],
+    ) -> Tuple[Optional[Dict], Optional[str]]:
+        """
+        Search for an item across multiple JSON files.
+
+        :param str identifier: Identifier to search for
+        :param List[str] file_paths: List of file paths to search
+        :param Callable match_fn: Function to determine if an item matches (takes item and identifier)
+        :return: Tuple of (found_item, source_file) or (None, None)
+        :rtype: Tuple[Optional[Dict], Optional[str]]
+        """
+        for file_path in file_paths:
+            if not os.path.exists(file_path):
+                continue
+
+            try:
+                result = FileOperations.search_single_json_file(identifier, file_path, match_fn)
+                if result:
+                    return result, file_path
+            except Exception as e:
+                logger.debug(f"Error searching {file_path}: {e}")
+                continue
+
+        return None, None
+
+    @staticmethod
+    def search_single_json_file(
+        identifier: str,
+        file_path: str,
+        match_fn: Callable[[Dict, str], bool],
+    ) -> Optional[Dict]:
+        """
+        Search for an item in a single JSON file.
+
+        :param str identifier: Identifier to search for
+        :param str file_path: Path to JSON file
+        :param Callable match_fn: Function to determine if an item matches
+        :return: Matched item or None
+        :rtype: Optional[Dict]
+        """
+        logger.debug(f"Searching for {identifier} in {file_path}")
+
+        data = FileOperations.load_json_file(file_path)
+        if not isinstance(data, list):
+            return None
+
+        # Use generator for memory efficiency
+        return next((item for item in data if match_fn(item, identifier)), None)
+
+    @staticmethod
+    def load_cached_findings(
+        query_configs: List[Dict[str, Any]],
+        progress_callback: Optional[Callable] = None,
+    ) -> List[Tuple[str, List[Dict], Optional[Exception]]]:
+        """
+        Load cached findings from multiple files.
+
+        :param List[Dict[str, Any]] query_configs: Query configurations with file paths
+        :param Optional[Callable] progress_callback: Optional progress callback
+        :return: List of (query_type, nodes, error) tuples
+        :rtype: List[Tuple[str, List[Dict], Optional[Exception]]]
+        """
+        results = []
+
+        for config in query_configs:
+            query_type = config["type"].value
+            file_path = config.get("file_path")
+
+            if progress_callback:
+                progress_callback(query_type, "loading")
+
+            try:
+                if file_path and os.path.exists(file_path):
+                    nodes = FileOperations.load_json_file(file_path)
+                    if nodes is not None:
+                        logger.info(f"Loaded {len(nodes)} cached {query_type} findings from {file_path}")
+                        results.append((query_type, nodes, None))
+                    else:
+                        logger.warning(f"Failed to load cached data for {query_type}")
+                        results.append((query_type, [], Exception(f"Failed to load {file_path}")))
+                else:
+                    logger.warning(f"No cached data found for {query_type} at {file_path}")
+                    results.append((query_type, [], None))
+
+            except Exception as e:
+                logger.error(f"Error loading cached data for {query_type}: {e}")
+                results.append((query_type, [], e))
+
+            if progress_callback:
+                progress_callback(query_type, "loaded")
+
+        return results

regscale/integrations/commercial/wizv2/fetchers/__init__.py

@@ -0,0 +1,11 @@
+"""Fetchers for Wiz integration - handles data retrieval and caching."""
+
+from regscale.integrations.commercial.wizv2.fetchers.policy_assessment import (
+    PolicyAssessmentFetcher,
+    WizDataCache,
+)
+
+__all__ = [
+    "PolicyAssessmentFetcher",
+    "WizDataCache",
+]