regscale-cli 6.20.4.1__py3-none-any.whl → 6.20.5.0__py3-none-any.whl

regscale/__init__.py

@@ -1 +1 @@
-__version__ = "6.20.4.1"
+__version__ = "6.20.5.0"

regscale/core/app/internal/model_editor.py

@@ -1365,7 +1365,7 @@ def build_object_field_list(obj: object) -> None:
     """
     # Build the list of fields for the model type
     pos_dict = obj.get_sort_position_dict()
-    field_names = obj.model_fields.keys()
+    field_names = obj.__class__.model_fields.keys()
     extra_fields = obj.get_extra_fields()
     include_field_list = obj.get_include_fields()
     for item in include_field_list:
@@ -1375,7 +1375,7 @@ def build_object_field_list(obj: object) -> None:
             field_makeup = FieldMakeup(
                 cur_field,
                 convert_property_to_column_label(cur_field),
-                get_field_data_type(obj.model_fields[cur_field]),
+                get_field_data_type(obj.__class__.model_fields[cur_field]),
             )
             field_makeup.sort_order = find_sort_pos(cur_field, pos_dict)
             field_makeup.enum_values = obj.get_enum_values(cur_field)
@@ -1444,7 +1444,7 @@ def is_field_required(obj: object, field_name: str) -> bool:
     :return: bool indicating if the field is required
     :rtype: bool
     """
-    field_info = obj.model_fields[field_name]
+    field_info = obj.__class__.model_fields[field_name]
     if field_info.annotation == dict:
         return True
     if field_info.annotation == int:

regscale/core/app/utils/regscale_utils.py

@@ -19,6 +19,8 @@ from regscale.core.app.utils.app_utils import convert_to_string, error_and_exit,
 from regscale.models import Data
 from regscale.models.regscale_models.modules import Modules
 
+from regscale.models.regscale_models.form_field_value import FormFieldValue
+
 logger = create_logger()
 
 
@@ -316,3 +318,38 @@ def create_properties(
 
     new_props = Property.batch_create(properties)
     return len(new_props) > 0
+
+
+def normalize_controlid(name: str) -> str:
+    """
+    Normalizes a control Id String
+    e.g. AC-01(02) -> ac-1.2
+         AC-01a.[02] -> ac-1.a.2
+
+    :param str name: Control Id String to normalize
+    :return: normalized Control Id String
+    :rtype: str
+    """
+    # AC-01(02)
+    # AC-01a.[02] vs. AC-1a.2
+    new_string = name.replace(" ", "")
+    new_string = new_string.replace("(", ".")  # AC-01.02) #AC-01a.[02]
+    new_string = new_string.replace(")", "")  # AC-01.02 #AC-01.a.[02]
+    new_string = new_string.replace("[", "")  # AC-01.02 #AC-01.a.02]
+    new_string = new_string.replace("]", "")  # AC-01.02 #AC-01.a.02
+
+    parts = new_string.split(".")
+    new_string = ""
+    for part in parts:
+        part = part.lstrip("0")
+        new_string += f"{part}."
+    new_string = new_string.rstrip(".")  # AC-01.2 #AC-01.a.2
+
+    parts = new_string.split("-")
+    new_string = ""
+    for part in parts:
+        part = part.lstrip("0")
+        new_string += f"{part}-"
+    new_string = new_string.rstrip("-")  # AC-1.2 #AC-1.a.2
+
+    return new_string.lower()

regscale/core/utils/date.py

@@ -4,11 +4,10 @@
 
 import datetime
 import logging
-from typing import List, Union, Optional
+from typing import Any, List, Optional, Union
 
+from dateutil.parser import ParserError, parse
 from pandas import Timestamp
-from dateutil.parser import parse, ParserError
-
 
 logger = logging.getLogger("regscale")
 default_date_format = "%Y-%m-%dT%H:%M:%S%z"
@@ -236,3 +235,27 @@ def normalize_date(dt: str, fmt: str) -> str:
         except ValueError:
             return dt
     return dt
+
+
+def normalize_timestamp(timestamp_value: Any) -> int:
+    """
+    Normalize timestamp to seconds, handling both seconds and milliseconds.
+
+    :param Any timestamp_value: The timestamp value to normalize
+    :return: Timestamp in seconds
+    :raises ValueError: If the timestamp is invalid
+    :rtype: int
+    """
+    if isinstance(timestamp_value, str):
+        if not timestamp_value.isdigit():
+            raise ValueError(f"Invalid timestamp value: {timestamp_value}")
+        timestamp_int = int(timestamp_value)
+    elif isinstance(timestamp_value, (int, float)):
+        timestamp_int = int(timestamp_value)
+    else:
+        raise ValueError(f"Invalid timestamp value type: {type(timestamp_value)}, defaulting to current datetime")
+
+    # Determine if it's epoch seconds or milliseconds based on magnitude
+    if timestamp_int > 9999999999:  # Likely milliseconds (13+ digits)
+        return timestamp_int // 1000
+    return timestamp_int

regscale/integrations/commercial/defender.py

@@ -27,6 +27,7 @@ from regscale.core.app.utils.app_utils import (
     uncamel_case,
     save_data_to,
 )
+from regscale.models.app_models.click import NotRequiredIf
 from regscale.models import regscale_id, regscale_module, regscale_ssp_id, Asset, Component, File, Issue
 from regscale.models.integration_models.defender_data import DefenderData
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
@@ -128,6 +129,8 @@ def sync_cloud_resources(regscale_ssp_id: int):
     help="The name of the saved query to export from Microsoft Defender for Cloud resource graph queries.",
     prompt="Enter the name of the query to export",
     default=None,
+    cls=NotRequiredIf,
+    not_required_if=["all_queries"],
 )
 @click.option(
     "--no_upload",

regscale/integrations/commercial/qualys/__init__.py

@@ -7,41 +7,43 @@ import os
 import pprint
 import traceback
 from asyncio import sleep
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 from json import JSONDecodeError
-from typing import Optional, Any, Union, Tuple
+from pathlib import Path
+from typing import Any, Optional, Tuple, Union
 from urllib.parse import urljoin
 
 import click
 import requests
 import xmltodict
-from pathlib import Path
 from requests import Session
-from rich.progress import TaskID, Progress, TextColumn, BarColumn, SpinnerColumn, TimeElapsedColumn
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
 
 from regscale.core.app.utils.app_utils import (
     check_file_path,
     check_license,
+    create_progress_object,
+    error_and_exit,
     get_current_datetime,
     save_data_to,
-    error_and_exit,
-    create_progress_object,
 )
 from regscale.core.app.utils.file_utils import download_from_s3
+from regscale.integrations.commercial.qualys.containers import fetch_all_vulnerabilities
 from regscale.integrations.commercial.qualys.qualys_error_handler import QualysErrorHandler
 from regscale.integrations.commercial.qualys.scanner import QualysTotalCloudJSONLIntegration
 from regscale.integrations.commercial.qualys.variables import QualysVariables
 from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
 from regscale.integrations.variables import ScannerVariables
-from regscale.models import Asset, Search, regscale_models, Issue
-from regscale.models.app_models.click import regscale_ssp_id, save_output_to, NotRequiredIf
+from regscale.models import Asset, Issue, Search, regscale_models
+from regscale.models.app_models.click import NotRequiredIf, regscale_ssp_id, save_output_to
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
 from regscale.models.integration_models.qualys import (
+    Qualys,
     QualysContainerScansImporter,
-    QualysWasScansImporter,
     QualysPolicyScansImporter,
-    Qualys,
+    QualysWasScansImporter,
 )
+from regscale.validation.record import validate_regscale_object
 
 # Create logger for this module
 logger = logging.getLogger("regscale")
@@ -332,14 +334,29 @@ class FindingProgressTracker:
     required=False,
     help="Enable/disable SSL certificate verification for API calls.",
 )
+@click.option(
+    "--containers",
+    type=click.BOOL,
+    help="To disable fetching containers, use False. Defaults to True.",
+    default=True,
+)
 def import_total_cloud(
-    regscale_ssp_id: int, include_tags: str, exclude_tags: str, vulnerability_creation: str, ssl_verify: bool
+    regscale_ssp_id: int,
+    include_tags: str,
+    exclude_tags: str,
+    vulnerability_creation: str,
+    ssl_verify: bool,
+    containers: bool,
 ):
     """
     Import Qualys Total Cloud Assets and Vulnerabilities using JSONL scanner implementation.
 
     This command uses the JSONLScannerIntegration class for improved efficiency and memory management.
     """
+    if not validate_regscale_object(regscale_ssp_id, "securityplans"):
+        logger.warning("SSP #%i is not a valid RegScale Security Plan.", regscale_ssp_id)
+        return
+    containers_lst = []
     try:
         # Configure scanner variables and fetch data
         _configure_scanner_variables(vulnerability_creation, ssl_verify)
@@ -347,8 +364,14 @@ def import_total_cloud(
         if not response_data:
             return
 
+        if containers:
+            # Fetch containers and container findings
+            containers_lst = fetch_all_vulnerabilities()
+
         # Initialize and run integration
-        integration = _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify)
+        integration = _initialize_integration(
+            regscale_ssp_id, response_data, vulnerability_creation, ssl_verify, containers_lst
+        )
         _run_integration_import(integration)
 
         logger.info("Qualys Total Cloud data imported successfully with JSONL scanner.")
@@ -460,13 +483,14 @@ def _fetch_qualys_api_data(include_tags, exclude_tags):
         return None
 
 
-def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify):
+def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify, containers):
     """Initialize the scanner integration with appropriate settings.
 
     :param int regscale_ssp_id: RegScale SSP ID
     :param dict response_data: Parsed XML data from API
     :param str vulnerability_creation: Vulnerability creation mode
     :param bool ssl_verify: SSL verification setting
+    :param list containers: List of containers
     :return: Initialized integration object
     """
     # Build integration kwargs
@@ -475,6 +499,7 @@ def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creati
         "xml_data": response_data,
         "vulnerability_creation": vulnerability_creation or ScannerVariables.vulnerabilityCreation,
         "ssl_verify": ssl_verify if ssl_verify is not None else ScannerVariables.sslVerify,
+        "containers": containers,
     }
 
     # Add thread workers if available
@@ -634,7 +659,7 @@ def _count_jsonl_items(integration):
 
 def _create_progress_bar():
     """Create a progress bar that doesn't reset at 100%."""
-    from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn
+    from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
 
     class NonResettingProgress(Progress):
         def update(self, task_id, **fields):
@@ -1120,6 +1145,7 @@ def process_files_with_importer(
     :param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True.
     """
     import csv
+
     from openpyxl import Workbook
 
     if s3_bucket:

regscale/integrations/commercial/qualys/containers.py

@@ -0,0 +1,324 @@
+"""
+Container operations module for Qualys CS API integration.
+"""
+
+import logging
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import lru_cache
+from json import JSONDecodeError
+from typing import Dict, List, Optional
+from urllib.parse import urljoin
+
+from requests import RequestException
+from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+
+# Create logger for this module
+logger = logging.getLogger("regscale")
+
+
+@lru_cache(maxsize=1)
+def auth_cs_api() -> tuple[str, dict]:
+    """
+    Authenticate the Qualys CS API using form-based authentication
+
+    :return: A tuple of the base URL and a dictionary of headers
+    :rtype: tuple[str, dict]
+    """
+    from . import QUALYS_API, _get_config  # noqa: C0415
+
+    config = _get_config()
+    qualys_url = config.get("qualysUrl")
+    user = config.get("qualysUserName")
+    password = config.get("qualysPassword")
+
+    # Update headers to match the curl command
+    auth_headers = {"X-Requested-With": "RegScale CLI"}
+
+    # Prepare form data for authentication
+    auth_data = {"username": user, "password": password, "permissions": "true", "token": "true"}
+
+    try:
+        # Make authentication request
+        # https://gateway.qg3.apps.qualys.com/auth
+
+        if qualys_url:
+            base_url = qualys_url.replace("qualysguard", "gateway")
+        else:
+            base_url = qualys_url
+
+        auth_url = urljoin(base_url, "/auth")
+        response = QUALYS_API.post(url=auth_url, headers=auth_headers, data=auth_data)
+
+        if response.ok:
+            logger.info("Successfully authenticated with Qualys CS API")
+
+            # Parse the response to extract the JWT token
+            try:
+                response_text = response.content.decode("utf-8")
+                # The response should contain the JWT token
+                # You might need to parse JSON or extract the token from the response
+                # For now, let's assume the token is in the response text
+
+                # Add Authorization Bearer header
+                auth_headers["Authorization"] = f"Bearer {response_text}"
+
+                logger.debug("Added Authorization Bearer header to auth_headers")
+            except (UnicodeDecodeError, AttributeError) as e:
+                logger.warning("Could not decode response content for Authorization header: %s", e)
+                logger.debug(
+                    "Response content type: %s, length: %s",
+                    type(response.content),
+                    len(response.content) if hasattr(response.content, "__len__") else "unknown",
+                )
+                # Continue without Authorization header if parsing fails
+        else:
+            raise RequestException(f"Authentication failed with status code: {response.status_code}")
+
+    except Exception as e:
+        logger.error("Error during authentication: %s", e)
+        raise
+
+    return base_url, auth_headers
+
+
+def _make_api_request(current_url: str, headers: dict, params: Optional[Dict] = None) -> dict:
+    """
+    Make API request to fetch containers from Qualys CS API
+
+    :param str current_url: The URL for the API request
+    :param dict headers: Headers to include in the request
+    :param Dict params: Optional query parameters for pagination
+    :return: Response data containing containers and response object
+    :rtype: dict
+    """
+    from . import QUALYS_API  # noqa: C0415
+
+    # Make API request
+    response = QUALYS_API.get(url=current_url, headers=headers, params=params)
+
+    # Validate response
+    if not response.ok:
+        logger.error("API request failed: %s - %s", response.status_code, response.text)
+        return {"data": [], "_response": response}
+
+    try:
+        response_data = response.json()
+        response_data["_response"] = response  # Include response object for headers
+        return response_data
+    except JSONDecodeError as e:
+        logger.error("Failed to parse JSON response: %s", e)
+        return {"data": [], "_response": response}
+
+
+def _parse_link_header(link_header: str) -> Optional[str]:
+    """
+    Parse the Link header to find the next page URL.
+
+    :param str link_header: The Link header value
+    :return: The next page URL or None if not found
+    :rtype: Optional[str]
+    """
+    if not link_header:
+        logger.debug("No Link header found, assuming no more pages")
+        return None
+
+    # Parse the Link header to find the next page URL
+    # Format: <url>;rel=next
+    for link in link_header.split(","):
+        link = link.strip()
+        if "rel=next" in link:
+            # Extract URL from <url>;rel=next format
+            url_start = link.find("<") + 1
+            url_end = link.find(">")
+            if 0 < url_start < url_end:
+                return link[url_start:url_end]
+
+    logger.debug("No next page URL found in Link header")
+    return None
+
+
+def _fetch_paginated_data(endpoint: str, filters: Optional[Dict] = None, limit: int = 100) -> List[Dict]:
+    """
+    Generic function to fetch paginated data from Qualys CS API
+
+    :param str endpoint: The API endpoint (e.g., 'containers/list', 'images/list')
+    :param Optional[Dict] filters: Filters to apply to the request
+    :param int limit: Number of items to fetch per page
+    :return: A list of items from all pages
+    :rtype: List[Dict]
+    """
+    all_items = []
+    page: int = 1
+
+    try:
+        # Get authentication
+        base_url, headers = auth_cs_api()
+
+        # Prepare base parameters
+        params = {"limit": limit}
+
+        # Add filters if provided
+        if filters:
+            params.update(filters)
+
+        # Track the current URL for pagination
+        current_url = urljoin(base_url, f"/csapi/v1.3/{endpoint}")
+
+        # Create progress bar for pagination
+        progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold blue]{task.description}"),
+            BarColumn(),
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+            TextColumn("•"),
+            TimeElapsedColumn(),
+            console=None,
+        )
+
+        with progress:
+            task = progress.add_task(f"[green]Fetching {endpoint} data...", total=None)  # Unknown total for pagination
+
+            while current_url:
+                # Make API request
+                response_data = _make_api_request(current_url, headers, params)
+
+                # Extract items from current page
+                current_items = response_data.get("data", [])
+                all_items.extend(current_items)
+
+                # Update progress description with current status
+                progress.update(
+                    task, description=f"[green]Fetching {endpoint} data... (Page {page}, Total: {len(all_items)})"
+                )
+
+                logger.debug("Fetched page: %s items (Total so far: %s)", page, len(all_items))
+
+                # Check for next page using the Link header
+                response = response_data.get("_response")
+                if not response or not hasattr(response, "headers"):
+                    # If no response object available, assume single page
+                    break
+
+                link_header = response.headers.get("link", "")
+                next_url = _parse_link_header(link_header)
+
+                if not next_url:
+                    break
+
+                # Update current URL for next iteration
+                current_url = next_url
+                page += 1
+
+                # Clear params for subsequent requests since they're in the URL
+                params = {}
+
+    except Exception as e:
+        logger.error("Error fetching data from %s: %s", current_url, e)
+        logger.debug(traceback.format_exc())
+
+    logger.info("Completed: Fetched %s total items from %s", len(all_items), endpoint)
+    return all_items
+
+
+def fetch_all_containers(filters: Optional[Dict] = None, limit: int = 100) -> List[Dict]:
+    """
+    Fetch all containers from Qualys CS API with pagination
+
+    :param Optional[Dict] filters: Filters to apply to the containers
+    :param int limit: Number of containers to fetch per page
+    :return: A list of containers
+    :rtype: List[Dict]
+    """
+    return _fetch_paginated_data("containers/list", filters, limit)
+
+
+def fetch_all_images(filters: Optional[Dict] = None, limit: int = 100) -> List[Dict]:
+    """
+    Fetch all images from Qualys CS API with pagination
+
+    :param Optional[Dict] filters: Filters to apply to the images
+    :param int limit: Number of images to fetch per page
+    :return: A list of images
+    :rtype: List[Dict]
+    """
+    return _fetch_paginated_data("images/list", filters, limit)
+
+
+def fetch_container_vulns(container_sha: str) -> List[Dict]:
+    """
+    Fetch vulnerabilities for a specific container from Qualys CS API
+
+    :param str container_sha: The SHA of the container
+    :return: A list of vulnerabilities
+    :rtype: List[Dict]
+    """
+    base_url, headers = auth_cs_api()
+    current_url = urljoin(base_url, f"/csapi/v1.3/containers/{container_sha}/vuln")
+    response_data = _make_api_request(current_url, headers)
+    return response_data.get("details", {}).get("vulns", [])
+
+
+def fetch_all_vulnerabilities(filters: Optional[Dict] = None, limit: int = 100, max_workers: int = 10) -> List[Dict]:
+    """
+    Fetch all containers and a list of vulnerabilities for each container from Qualys CS API with pagination
+
+    :param Optional[Dict] filters: Filters to apply to the containers
+    :param int limit: Number of containers to fetch per page
+    :param int max_workers: Maximum number of worker threads for concurrent vulnerability fetching
+    :return: A list of containers with vulnerabilities
+    :rtype: List[Dict]
+    """
+    containers = fetch_all_containers(filters, limit)
+
+    if not containers:
+        logger.info("No containers found to fetch vulnerabilities for")
+        return containers
+
+    # Create progress bar for fetching vulnerabilities
+    progress = Progress(
+        SpinnerColumn(),
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        TextColumn("•"),
+        TimeElapsedColumn(),
+        console=None,
+    )
+
+    def fetch_container_vulns_with_progress(container):
+        """Helper function to fetch vulnerabilities for a single container with progress tracking."""
+        container_sha = container.get("sha")
+        if not container_sha:
+            logger.warning("Container missing SHA, skipping vulnerability fetch")
+            return container, []
+
+        try:
+            vulns = fetch_container_vulns(container_sha)
+            logger.debug("Fetched %s vulnerabilities for container %s...", len(vulns), container_sha[:8])
+            return container, vulns
+        except Exception as e:
+            logger.error("Error fetching vulnerabilities for container %s: %s", container_sha, e)
+            return container, []
+
+    with progress:
+        task = progress.add_task(
+            f"[yellow]Fetching vulnerabilities for {len(containers)} containers...", total=len(containers)
+        )
+
+        # Use ThreadPoolExecutor for concurrent processing
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all tasks
+            future_to_container = {}
+            for container in containers:
+                future = executor.submit(fetch_container_vulns_with_progress, container)
+                future_to_container[future] = container
+
+            # Process completed tasks and update progress
+            for future in as_completed(future_to_container):
+                container, vulns = future.result()
+                container["vulnerabilities"] = vulns
+                progress.update(task, advance=1)
+
+    logger.info("Completed fetching vulnerabilities for %s containers using %s workers", len(containers), max_workers)
+    return containers