regscale-cli 6.20.4.1__py3-none-any.whl → 6.20.6.0__py3-none-any.whl

regscale/integrations/commercial/qualys/__init__.py

@@ -7,41 +7,43 @@ import os
 import pprint
 import traceback
 from asyncio import sleep
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 from json import JSONDecodeError
-from typing import Optional, Any, Union, Tuple
+from pathlib import Path
+from typing import Any, Optional, Tuple, Union
 from urllib.parse import urljoin
 
 import click
 import requests
 import xmltodict
-from pathlib import Path
 from requests import Session
-from rich.progress import TaskID, Progress, TextColumn, BarColumn, SpinnerColumn, TimeElapsedColumn
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
 
 from regscale.core.app.utils.app_utils import (
     check_file_path,
     check_license,
+    create_progress_object,
+    error_and_exit,
     get_current_datetime,
     save_data_to,
-    error_and_exit,
-    create_progress_object,
 )
 from regscale.core.app.utils.file_utils import download_from_s3
+from regscale.integrations.commercial.qualys.containers import fetch_all_vulnerabilities
 from regscale.integrations.commercial.qualys.qualys_error_handler import QualysErrorHandler
 from regscale.integrations.commercial.qualys.scanner import QualysTotalCloudJSONLIntegration
 from regscale.integrations.commercial.qualys.variables import QualysVariables
 from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
 from regscale.integrations.variables import ScannerVariables
-from regscale.models import Asset, Search, regscale_models, Issue
-from regscale.models.app_models.click import regscale_ssp_id, save_output_to, NotRequiredIf
+from regscale.models import Asset, Issue, Search, regscale_models
+from regscale.models.app_models.click import NotRequiredIf, regscale_ssp_id, save_output_to
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
 from regscale.models.integration_models.qualys import (
+    Qualys,
     QualysContainerScansImporter,
-    QualysWasScansImporter,
     QualysPolicyScansImporter,
-    Qualys,
+    QualysWasScansImporter,
 )
+from regscale.validation.record import validate_regscale_object
 
 # Create logger for this module
 logger = logging.getLogger("regscale")
@@ -332,14 +334,29 @@ class FindingProgressTracker:
     required=False,
     help="Enable/disable SSL certificate verification for API calls.",
 )
+@click.option(
+    "--containers",
+    type=click.BOOL,
+    help="To disable fetching containers, use False. Defaults to True.",
+    default=True,
+)
 def import_total_cloud(
-    regscale_ssp_id: int, include_tags: str, exclude_tags: str, vulnerability_creation: str, ssl_verify: bool
+    regscale_ssp_id: int,
+    include_tags: str,
+    exclude_tags: str,
+    vulnerability_creation: str,
+    ssl_verify: bool,
+    containers: bool,
 ):
     """
     Import Qualys Total Cloud Assets and Vulnerabilities using JSONL scanner implementation.
 
     This command uses the JSONLScannerIntegration class for improved efficiency and memory management.
     """
+    if not validate_regscale_object(regscale_ssp_id, "securityplans"):
+        logger.warning("SSP #%i is not a valid RegScale Security Plan.", regscale_ssp_id)
+        return
+    containers_lst = []
     try:
         # Configure scanner variables and fetch data
         _configure_scanner_variables(vulnerability_creation, ssl_verify)
@@ -347,8 +364,14 @@ def import_total_cloud(
         if not response_data:
             return
 
+        if containers:
+            # Fetch containers and container findings
+            containers_lst = fetch_all_vulnerabilities()
+
         # Initialize and run integration
-        integration = _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify)
+        integration = _initialize_integration(
+            regscale_ssp_id, response_data, vulnerability_creation, ssl_verify, containers_lst
+        )
         _run_integration_import(integration)
 
         logger.info("Qualys Total Cloud data imported successfully with JSONL scanner.")
@@ -460,13 +483,14 @@ def _fetch_qualys_api_data(include_tags, exclude_tags):
         return None
 
 
-def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify):
+def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify, containers):
     """Initialize the scanner integration with appropriate settings.
 
     :param int regscale_ssp_id: RegScale SSP ID
     :param dict response_data: Parsed XML data from API
     :param str vulnerability_creation: Vulnerability creation mode
     :param bool ssl_verify: SSL verification setting
+    :param list containers: List of containers
     :return: Initialized integration object
     """
     # Build integration kwargs
@@ -475,6 +499,7 @@ def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creati
         "xml_data": response_data,
         "vulnerability_creation": vulnerability_creation or ScannerVariables.vulnerabilityCreation,
         "ssl_verify": ssl_verify if ssl_verify is not None else ScannerVariables.sslVerify,
+        "containers": containers,
     }
 
     # Add thread workers if available
@@ -634,7 +659,7 @@ def _count_jsonl_items(integration):
 
 def _create_progress_bar():
     """Create a progress bar that doesn't reset at 100%."""
-    from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn
+    from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
 
     class NonResettingProgress(Progress):
         def update(self, task_id, **fields):
@@ -1120,6 +1145,7 @@ def process_files_with_importer(
     :param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True.
     """
     import csv
+
     from openpyxl import Workbook
 
     if s3_bucket:

regscale/integrations/commercial/qualys/containers.py

@@ -0,0 +1,324 @@
+"""
+Container operations module for Qualys CS API integration.
+"""
+
+import logging
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import lru_cache
+from json import JSONDecodeError
+from typing import Dict, List, Optional
+from urllib.parse import urljoin
+
+from requests import RequestException
+from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+
+# Create logger for this module
+logger = logging.getLogger("regscale")
+
+
+@lru_cache(maxsize=1)
+def auth_cs_api() -> tuple[str, dict]:
+    """
+    Authenticate the Qualys CS API using form-based authentication
+
+    :return: A tuple of the base URL and a dictionary of headers
+    :rtype: tuple[str, dict]
+    """
+    from . import QUALYS_API, _get_config  # noqa: C0415
+
+    config = _get_config()
+    qualys_url = config.get("qualysUrl")
+    user = config.get("qualysUserName")
+    password = config.get("qualysPassword")
+
+    # Update headers to match the curl command
+    auth_headers = {"X-Requested-With": "RegScale CLI"}
+
+    # Prepare form data for authentication
+    auth_data = {"username": user, "password": password, "permissions": "true", "token": "true"}
+
+    try:
+        # Make authentication request
+        # https://gateway.qg3.apps.qualys.com/auth
+
+        if qualys_url:
+            base_url = qualys_url.replace("qualysguard", "gateway")
+        else:
+            base_url = qualys_url
+
+        auth_url = urljoin(base_url, "/auth")
+        response = QUALYS_API.post(url=auth_url, headers=auth_headers, data=auth_data)
+
+        if response.ok:
+            logger.info("Successfully authenticated with Qualys CS API")
+
+            # Parse the response to extract the JWT token
+            try:
+                response_text = response.content.decode("utf-8")
+                # The response should contain the JWT token
+                # You might need to parse JSON or extract the token from the response
+                # For now, let's assume the token is in the response text
+
+                # Add Authorization Bearer header
+                auth_headers["Authorization"] = f"Bearer {response_text}"
+
+                logger.debug("Added Authorization Bearer header to auth_headers")
+            except (UnicodeDecodeError, AttributeError) as e:
+                logger.warning("Could not decode response content for Authorization header: %s", e)
+                logger.debug(
+                    "Response content type: %s, length: %s",
+                    type(response.content),
+                    len(response.content) if hasattr(response.content, "__len__") else "unknown",
+                )
+                # Continue without Authorization header if parsing fails
+        else:
+            raise RequestException(f"Authentication failed with status code: {response.status_code}")
+
+    except Exception as e:
+        logger.error("Error during authentication: %s", e)
+        raise
+
+    return base_url, auth_headers
+
+
+def _make_api_request(current_url: str, headers: dict, params: Optional[Dict] = None) -> dict:
+    """
+    Make API request to fetch containers from Qualys CS API
+
+    :param str current_url: The URL for the API request
+    :param dict headers: Headers to include in the request
+    :param Dict params: Optional query parameters for pagination
+    :return: Response data containing containers and response object
+    :rtype: dict
+    """
+    from . import QUALYS_API  # noqa: C0415
+
+    # Make API request
+    response = QUALYS_API.get(url=current_url, headers=headers, params=params)
+
+    # Validate response
+    if not response.ok:
+        logger.error("API request failed: %s - %s", response.status_code, response.text)
+        return {"data": [], "_response": response}
+
+    try:
+        response_data = response.json()
+        response_data["_response"] = response  # Include response object for headers
+        return response_data
+    except JSONDecodeError as e:
+        logger.error("Failed to parse JSON response: %s", e)
+        return {"data": [], "_response": response}
+
+
+def _parse_link_header(link_header: str) -> Optional[str]:
+    """
+    Parse the Link header to find the next page URL.
+
+    :param str link_header: The Link header value
+    :return: The next page URL or None if not found
+    :rtype: Optional[str]
+    """
+    if not link_header:
+        logger.debug("No Link header found, assuming no more pages")
+        return None
+
+    # Parse the Link header to find the next page URL
+    # Format: <url>;rel=next
+    for link in link_header.split(","):
+        link = link.strip()
+        if "rel=next" in link:
+            # Extract URL from <url>;rel=next format
+            url_start = link.find("<") + 1
+            url_end = link.find(">")
+            if 0 < url_start < url_end:
+                return link[url_start:url_end]
+
+    logger.debug("No next page URL found in Link header")
+    return None
+
+
+def _fetch_paginated_data(endpoint: str, filters: Optional[Dict] = None, limit: int = 100) -> List[Dict]:
+    """
+    Generic function to fetch paginated data from Qualys CS API
+
+    :param str endpoint: The API endpoint (e.g., 'containers/list', 'images/list')
+    :param Optional[Dict] filters: Filters to apply to the request
+    :param int limit: Number of items to fetch per page
+    :return: A list of items from all pages
+    :rtype: List[Dict]
+    """
+    all_items = []
+    page: int = 1
+
+    try:
+        # Get authentication
+        base_url, headers = auth_cs_api()
+
+        # Prepare base parameters
+        params = {"limit": limit}
+
+        # Add filters if provided
+        if filters:
+            params.update(filters)
+
+        # Track the current URL for pagination
+        current_url = urljoin(base_url, f"/csapi/v1.3/{endpoint}")
+
+        # Create progress bar for pagination
+        progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold blue]{task.description}"),
+            BarColumn(),
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+            TextColumn("•"),
+            TimeElapsedColumn(),
+            console=None,
+        )
+
+        with progress:
+            task = progress.add_task(f"[green]Fetching {endpoint} data...", total=None)  # Unknown total for pagination
+
+            while current_url:
+                # Make API request
+                response_data = _make_api_request(current_url, headers, params)
+
+                # Extract items from current page
+                current_items = response_data.get("data", [])
+                all_items.extend(current_items)
+
+                # Update progress description with current status
+                progress.update(
+                    task, description=f"[green]Fetching {endpoint} data... (Page {page}, Total: {len(all_items)})"
+                )
+
+                logger.debug("Fetched page: %s items (Total so far: %s)", page, len(all_items))
+
+                # Check for next page using the Link header
+                response = response_data.get("_response")
+                if not response or not hasattr(response, "headers"):
+                    # If no response object available, assume single page
+                    break
+
+                link_header = response.headers.get("link", "")
+                next_url = _parse_link_header(link_header)
+
+                if not next_url:
+                    break
+
+                # Update current URL for next iteration
+                current_url = next_url
+                page += 1
+
+                # Clear params for subsequent requests since they're in the URL
+                params = {}
+
+    except Exception as e:
+        logger.error("Error fetching data from %s: %s", current_url, e)
+        logger.debug(traceback.format_exc())
+
+    logger.info("Completed: Fetched %s total items from %s", len(all_items), endpoint)
+    return all_items
+
+
+def fetch_all_containers(filters: Optional[Dict] = None, limit: int = 100) -> List[Dict]:
+    """
+    Fetch all containers from Qualys CS API with pagination
+
+    :param Optional[Dict] filters: Filters to apply to the containers
+    :param int limit: Number of containers to fetch per page
+    :return: A list of containers
+    :rtype: List[Dict]
+    """
+    return _fetch_paginated_data("containers/list", filters, limit)
+
+
+def fetch_all_images(filters: Optional[Dict] = None, limit: int = 100) -> List[Dict]:
+    """
+    Fetch all images from Qualys CS API with pagination
+
+    :param Optional[Dict] filters: Filters to apply to the images
+    :param int limit: Number of images to fetch per page
+    :return: A list of images
+    :rtype: List[Dict]
+    """
+    return _fetch_paginated_data("images/list", filters, limit)
+
+
+def fetch_container_vulns(container_sha: str) -> List[Dict]:
+    """
+    Fetch vulnerabilities for a specific container from Qualys CS API
+
+    :param str container_sha: The SHA of the container
+    :return: A list of vulnerabilities
+    :rtype: List[Dict]
+    """
+    base_url, headers = auth_cs_api()
+    current_url = urljoin(base_url, f"/csapi/v1.3/containers/{container_sha}/vuln")
+    response_data = _make_api_request(current_url, headers)
+    return response_data.get("details", {}).get("vulns", [])
+
+
+def fetch_all_vulnerabilities(filters: Optional[Dict] = None, limit: int = 100, max_workers: int = 10) -> List[Dict]:
+    """
+    Fetch all containers and a list of vulnerabilities for each container from Qualys CS API with pagination
+
+    :param Optional[Dict] filters: Filters to apply to the containers
+    :param int limit: Number of containers to fetch per page
+    :param int max_workers: Maximum number of worker threads for concurrent vulnerability fetching
+    :return: A list of containers with vulnerabilities
+    :rtype: List[Dict]
+    """
+    containers = fetch_all_containers(filters, limit)
+
+    if not containers:
+        logger.info("No containers found to fetch vulnerabilities for")
+        return containers
+
+    # Create progress bar for fetching vulnerabilities
+    progress = Progress(
+        SpinnerColumn(),
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        TextColumn("•"),
+        TimeElapsedColumn(),
+        console=None,
+    )
+
+    def fetch_container_vulns_with_progress(container):
+        """Helper function to fetch vulnerabilities for a single container with progress tracking."""
+        container_sha = container.get("sha")
+        if not container_sha:
+            logger.warning("Container missing SHA, skipping vulnerability fetch")
+            return container, []
+
+        try:
+            vulns = fetch_container_vulns(container_sha)
+            logger.debug("Fetched %s vulnerabilities for container %s...", len(vulns), container_sha[:8])
+            return container, vulns
+        except Exception as e:
+            logger.error("Error fetching vulnerabilities for container %s: %s", container_sha, e)
+            return container, []
+
+    with progress:
+        task = progress.add_task(
+            f"[yellow]Fetching vulnerabilities for {len(containers)} containers...", total=len(containers)
+        )
+
+        # Use ThreadPoolExecutor for concurrent processing
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all tasks
+            future_to_container = {}
+            for container in containers:
+                future = executor.submit(fetch_container_vulns_with_progress, container)
+                future_to_container[future] = container
+
+            # Process completed tasks and update progress
+            for future in as_completed(future_to_container):
+                container, vulns = future.result()
+                container["vulnerabilities"] = vulns
+                progress.update(task, advance=1)
+
+    logger.info("Completed fetching vulnerabilities for %s containers using %s workers", len(containers), max_workers)
+    return containers