regscale-cli 6.17.0.0__py3-none-any.whl → 6.19.0.0__py3-none-any.whl

regscale/integrations/commercial/{qualys.py → qualys/__init__.py}

@@ -1,13 +1,15 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Integrates Qualys assets and vulnerabilities into RegScale CLI"""
+"""
+Qualys integration module for RegScale CLI.
+"""
+
+import logging
 import os
 import pprint
 import traceback
 from asyncio import sleep
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone, timedelta
 from json import JSONDecodeError
-from typing import Any, Optional, Tuple, Union
+from typing import Optional, Any, Union, Tuple
 from urllib.parse import urljoin
 
 import click
@@ -15,53 +17,678 @@ import requests
 import xmltodict
 from pathlib import Path
 from requests import Session
-from rich.progress import TaskID
+from rich.progress import TaskID, Progress, TextColumn, BarColumn, SpinnerColumn, TimeElapsedColumn
 
-from regscale.core.app.logz import create_logger
 from regscale.core.app.utils.app_utils import (
     check_file_path,
     check_license,
-    create_progress_object,
-    error_and_exit,
     get_current_datetime,
     save_data_to,
+    error_and_exit,
+    create_progress_object,
 )
 from regscale.core.app.utils.file_utils import download_from_s3
-from regscale.models import Asset, Issue, Search, regscale_models
-from regscale.models.app_models.click import NotRequiredIf, save_output_to
-from regscale.models.app_models.click import regscale_ssp_id
+from regscale.integrations.commercial.qualys.scanner import QualysTotalCloudJSONLIntegration
+from regscale.integrations.commercial.qualys.variables import QualysVariables
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
+from regscale.integrations.variables import ScannerVariables
+from regscale.models import Asset, Search, regscale_models, Issue
+from regscale.models.app_models.click import regscale_ssp_id, save_output_to, NotRequiredIf
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
 from regscale.models.integration_models.qualys import (
-    Qualys,
     QualysContainerScansImporter,
     QualysWasScansImporter,
     QualysPolicyScansImporter,
+    Qualys,
 )
-from regscale.models.integration_models.qualys_scanner import QualysTotalCloudIntegration
-
-####################################################################################################
-#
-# Qualys API Documentation:
-#   https://qualysguard.qg2.apps.qualys.com/qwebhelp/fo_portal/api_doc/index.htm
-#
-####################################################################################################
-
 
-# create global variables for the entire module
-logger = create_logger()
-
-# create progress object to add tasks to for real time updates
+# Create logger for this module
+logger = logging.getLogger("regscale")
 job_progress = create_progress_object()
 HEADERS = {"X-Requested-With": "RegScale CLI"}
+
+# Import the Qualys API session object and headers from the main qualys module
 QUALYS_API = Session()
 
 
-# Create group to handle Qualys commands
 @click.group()
 def qualys():
     """Performs actions from the Qualys API"""
 
 
+def _prepare_qualys_params(include_tags: str, exclude_tags: str) -> dict:
+    """
+    Prepare parameters for Qualys API request.
+
+    :param str include_tags: Tags to include in the filter
+    :param str exclude_tags: Tags to exclude in the filter
+    :return: Dictionary of parameters for the API request
+    :rtype: dict
+    """
+    params = {
+        "action": "list",
+        "show_asset_id": "1",
+        "show_tags": "1",
+    }
+
+    if exclude_tags or include_tags:
+        params["use_tags"] = "1"
+        if exclude_tags:
+            params["tag_set_exclude"] = exclude_tags
+        if include_tags:
+            params["tag_set_include"] = include_tags
+
+    return params
+
+
+def _setup_progress_tracking(integration, progress):
+    """
+    Set up progress tracking for assets and findings.
+
+    :param integration: Scanner integration instance
+    :param progress: Progress bar instance
+    :return: Tuple of task IDs and counts for assets and findings
+    :rtype: tuple
+    """
+    # Count assets and findings for progress tracking
+    asset_count = sum(1 for _ in open(integration.ASSETS_FILE, "r") if _.strip())
+    finding_count = sum(1 for _ in open(integration.FINDINGS_FILE, "r") if _.strip())
+
+    logger.info(f"Found {asset_count} assets and {finding_count} findings in JSONL files")
+
+    # Create tasks
+    asset_task = progress.add_task(f"[green]Importing {asset_count} assets to RegScale...", total=asset_count)
+    finding_task = progress.add_task(f"[yellow]Importing {finding_count} findings to RegScale...", visible=False)
+
+    return asset_task, finding_task, asset_count, finding_count
+
+
+def _import_assets(integration, assets_iterator, progress, asset_task):
+    """
+    Import assets to RegScale with progress tracking.
+
+    :param integration: Scanner integration instance
+    :param assets_iterator: Iterator of assets
+    :param progress: Progress bar instance to track progress
+    :param asset_task: Task ID for asset progress
+    :return: Number of assets imported
+    :rtype: int
+    """
+    if not _verify_assets_file_exists(integration):
+        return 0
+
+    try:
+        wrapped_iterator = _create_asset_progress_tracker(assets_iterator, progress, asset_task)
+        assets_imported = integration.update_regscale_assets(wrapped_iterator)
+        logger.info(f"Imported {assets_imported} assets to RegScale")
+        return assets_imported
+    except Exception as e:
+        logger.error(f"Error importing assets to RegScale: {str(e)}")
+        logger.error(traceback.format_exc())
+        return 0
+
+
+def _verify_assets_file_exists(integration):
+    """
+    Verify that the assets file exists and is not empty.
+
+    :param integration: Scanner integration instance
+    :return: True if the file exists and is not empty, False otherwise
+    :rtype: bool
+    """
+    if not os.path.exists(integration.ASSETS_FILE) or os.path.getsize(integration.ASSETS_FILE) == 0:
+        logger.warning(f"Assets file {integration.ASSETS_FILE} is empty or does not exist")
+        return False
+    return True
+
+
+def _create_asset_progress_tracker(assets_iter, progress, asset_task):
+    """
+    Create a generator that tracks progress of asset processing.
+
+    :param assets_iter: Iterator of assets
+    :param progress: Progress bar instance
+    :param asset_task: Task ID for asset progress
+    :return: Generator that yields assets and updates progress
+    """
+    count = 0
+    asset_ids = []
+
+    try:
+        for asset in assets_iter:
+            count += 1
+            if asset and hasattr(asset, "identifier"):
+                asset_ids.append(asset.identifier)
+            progress.update(asset_task, advance=1)
+            yield asset
+
+        _log_asset_results(count, asset_ids)
+    except Exception as e:
+        logger.error(f"Error while yielding assets: {str(e)}")
+        logger.error(traceback.format_exc())
+
+
+def _log_asset_results(count, asset_ids):
+    """
+    Log the results of asset processing.
+
+    :param count: Number of assets processed
+    :param asset_ids: List of asset IDs
+    """
+    if count == 0:
+        logger.warning("No assets were yielded from the JSONL file")
+    else:
+        sample_ids = asset_ids[:5]
+        truncation_indicator = ", ..." if len(asset_ids) > 5 else ""
+        sample_ids_str = ", ".join(sample_ids)
+        logger.debug(f"Yielded {count} assets to RegScale with IDs: {sample_ids_str}{truncation_indicator}")
+
+
+def _import_findings(integration, progress, finding_task):
+    """
+    Import findings to RegScale with progress tracking.
+
+    :param integration: Scanner integration instance
+    :param progress: Progress bar instance
+    :param finding_task: Task ID for finding progress
+    :return: Number of findings imported
+    :rtype: int
+    """
+    progress.update(finding_task, visible=True)
+    total_findings = _count_findings_in_file(integration)
+
+    if total_findings > 0:
+        progress.update(finding_task, total=total_findings)
+
+    findings_yielded = 0
+    try:
+        # Create findings iterator directly from the file
+        tracked_iterator = _create_finding_progress_tracker(
+            integration._yield_items_from_jsonl(integration.FINDINGS_FILE, IntegrationFinding), progress, finding_task
+        )
+
+        findings_imported = integration.update_regscale_findings(tracked_iterator)
+        logger.info(f"Successfully imported {findings_imported} findings to RegScale")
+
+        # Update final count for nonlocal reference from the tracked_iterator
+        findings_yielded = getattr(tracked_iterator, "count", 0)
+
+        # Ensure progress is complete
+        if findings_yielded > 0:
+            progress.update(finding_task, completed=findings_yielded)
+
+        return findings_imported
+    except Exception as e:
+        logger.error(f"Error during RegScale findings import: {str(e)}")
+        return 0
+
+
+def _count_findings_in_file(integration):
+    """
+    Count the findings in the JSONL file.
+
+    :param integration: Scanner integration instance
+    :return: Number of findings in the file
+    :rtype: int
+    """
+    try:
+        if os.path.exists(integration.FINDINGS_FILE):
+            with open(integration.FINDINGS_FILE, "r") as f:
+                total_findings = sum(1 for line in f if line.strip())
+                logger.info(f"Found {total_findings} findings in JSONL file")
+                return total_findings
+        else:
+            logger.warning(f"Findings file {integration.FINDINGS_FILE} does not exist")
+            return 0
+    except Exception as e:
+        logger.error(f"Error counting findings in JSONL file: {str(e)}")
+        return 0
+
+
+def _create_finding_progress_tracker(findings_iter, progress, finding_task):
+    """
+    Create a generator function that tracks progress of finding processing.
+
+    :param findings_iter: Iterator of findings
+    :param progress: Progress bar instance
+    :param finding_task: Task ID for finding progress
+    :return: Generator that yields findings and updates progress
+    """
+    tracker = FindingProgressTracker(findings_iter, progress, finding_task)
+    return tracker
+
+
+class FindingProgressTracker:
+    """Class to track progress of finding processing with proper object reference."""
+
+    def __init__(self, findings_iter, progress, finding_task):
+        self.findings_iter = findings_iter
+        self.progress = progress
+        self.finding_task = finding_task
+        self.count = 0
+        self.finding_ids = []
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            finding = next(self.findings_iter)
+            self.count += 1
+            if finding and hasattr(finding, "external_id"):
+                self.finding_ids.append(finding.external_id)
+            self.progress.update(self.finding_task, advance=1)
+            return finding
+        except StopIteration:
+            self._log_finding_results()
+            raise
+        except Exception as e:
+            logger.error(f"Error yielding findings: {str(e)}")
+            logger.debug(f"Findings yielded before error: {self.count}")
+            raise
+
+    def _log_finding_results(self):
+        """Log the results of finding processing."""
+        if self.count == 0:
+            logger.warning("No findings were yielded from the JSONL file")
+        else:
+            logger.info(f"Yielded {self.count} findings to RegScale")
+            sample_ids = self.finding_ids[:5]
+            truncation_indicator = ", ..." if len(self.finding_ids) > 5 else ""
+            sample_ids_str = ", ".join(sample_ids)
+            logger.debug(f"Sample finding IDs: {sample_ids_str}{truncation_indicator}")
+
+
+@click.command(name="import_total_cloud")
+@regscale_ssp_id()
+@click.option(
+    "--include_tags",
+    "-t",
+    type=click.STRING,
+    required=False,
+    default=None,
+    help="Include tags in the import comma separated string of tag names or ids, defaults to None.",
+)
+@click.option(
+    "--exclude_tags",
+    "-e",
+    type=click.STRING,
+    required=False,
+    default=None,
+    help="Exclude tags in the import comma separated string of tag names or ids, defaults to None.",
+)
+@click.option(
+    "--vulnerability-creation",
+    "-v",
+    type=click.Choice(["NoIssue", "IssueCreation", "PoamCreation"], case_sensitive=False),
+    required=False,
+    default=None,
+    help="Specify how vulnerabilities are processed: NoIssue, IssueCreation, or PoamCreation.",
+)
+@click.option(
+    "--ssl-verify/--no-ssl-verify",
+    default=None,
+    required=False,
+    help="Enable/disable SSL certificate verification for API calls.",
+)
+def import_total_cloud(
+    regscale_ssp_id: int, include_tags: str, exclude_tags: str, vulnerability_creation: str, ssl_verify: bool
+):
+    """
+    Import Qualys Total Cloud Assets and Vulnerabilities using JSONL scanner implementation.
+
+    This command uses the JSONLScannerIntegration class for improved efficiency and memory management.
+    """
+    try:
+        # Configure scanner variables and fetch data
+        _configure_scanner_variables(vulnerability_creation, ssl_verify)
+        response_data = _fetch_qualys_api_data(include_tags, exclude_tags)
+        if not response_data:
+            return
+
+        # Initialize and run integration
+        integration = _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify)
+        _run_integration_import(integration)
+
+        logger.info("Qualys Total Cloud data imported successfully with JSONL scanner.")
+    except Exception:
+        error_message = traceback.format_exc()
+        logger.error("Error occurred while processing Qualys data with JSONL scanner")
+        logger.error(error_message)
+
+
+def _configure_scanner_variables(vulnerability_creation, ssl_verify):
+    """Configure scanner variables with appropriate precedence.
+
+    :param str vulnerability_creation: Vulnerability creation mode from command line
+    :param bool ssl_verify: SSL verification setting from command line
+    """
+    # Configure vulnerability creation mode
+    _configure_vulnerability_creation(vulnerability_creation)
+
+    # Configure SSL verification
+    _configure_ssl_verification(ssl_verify)
+
+
+def _configure_vulnerability_creation(vulnerability_creation):
+    """Configure vulnerability creation mode with appropriate precedence.
+
+    :param str vulnerability_creation: Vulnerability creation mode from command line
+    """
+    if vulnerability_creation:
+        # Command line option takes precedence
+        ScannerVariables.vulnerabilityCreation = vulnerability_creation
+        logger.info(f"Setting vulnerability creation mode from command line: {vulnerability_creation}")
+    elif hasattr(QualysVariables, "vulnerabilityCreation") and ScannerVariables.vulnerabilityCreation:
+        # Use Qualys-specific setting if available
+        logger.info(f"Using Qualys-specific vulnerability creation mode: {ScannerVariables.vulnerabilityCreation}")
+    else:
+        # Fall back to global ScannerVariables
+        logger.info(f"Using global vulnerability creation mode: {ScannerVariables.vulnerabilityCreation}")
+
+
+def _configure_ssl_verification(ssl_verify):
+    """Configure SSL verification setting with appropriate precedence.
+
+    :param bool ssl_verify: SSL verification setting from command line
+    """
+    if ssl_verify is not None:
+        # Command line option takes precedence
+        ScannerVariables.sslVerify = ssl_verify
+        logger.info(f"Setting SSL verification from command line: {ssl_verify}")
+    elif hasattr(QualysVariables, "sslVerify") and QualysVariables.sslVerify is not None:
+        # Use Qualys-specific setting
+        logger.info(f"Using Qualys-specific SSL verification setting: {ScannerVariables.sslVerify}")
+    else:
+        # Fall back to global ScannerVariables
+        logger.info(f"Using global SSL verification setting: {ScannerVariables.sslVerify}")
+
+
+def _fetch_qualys_api_data(include_tags, exclude_tags):
+    """Fetch data from Qualys API.
+
+    :param str include_tags: Tags to include in the query
+    :param str exclude_tags: Tags to exclude from the query
+    :return: Parsed XML data or None if request failed
+    """
+    qualys_url, qualys_api = _get_qualys_api()
+    params = _prepare_qualys_params(include_tags, exclude_tags)
+
+    logger.info("Fetching Qualys Total Cloud data with JSONL scanner...")
+    response = qualys_api.get(
+        url=urljoin(qualys_url, "/api/2.0/fo/asset/host/vm/detection/"),
+        headers=HEADERS,
+        params=params,
+        verify=ScannerVariables.sslVerify,  # Apply SSL verification setting
+    )
+
+    if not response or not response.ok:
+        logger.error(
+            f"Received unexpected response from Qualys API: {response.status_code}: {response.text if response.text else 'response is null'}"
+        )
+        return None
+
+    # Process API response
+    logger.info("Total cloud data fetched. Processing with JSONL scanner...")
+    return xmltodict.parse(response.text)
+
+
+def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify):
+    """Initialize the scanner integration with appropriate settings.
+
+    :param int regscale_ssp_id: RegScale SSP ID
+    :param dict response_data: Parsed XML data from API
+    :param str vulnerability_creation: Vulnerability creation mode
+    :param bool ssl_verify: SSL verification setting
+    :return: Initialized integration object
+    """
+    # Build integration kwargs
+    integration_kwargs = {
+        "plan_id": regscale_ssp_id,
+        "xml_data": response_data,
+        "vulnerability_creation": vulnerability_creation or ScannerVariables.vulnerabilityCreation,
+        "ssl_verify": ssl_verify if ssl_verify is not None else ScannerVariables.sslVerify,
+    }
+
+    # Add thread workers if available
+    if hasattr(ScannerVariables, "threadMaxWorkers"):
+        integration_kwargs["max_workers"] = ScannerVariables.threadMaxWorkers
+        logger.debug(f"Using thread max workers: {ScannerVariables.threadMaxWorkers}")
+
+    # Initialize and return integration
+    integration = QualysTotalCloudJSONLIntegration(**integration_kwargs)
+    return integration
+
+
+def _run_integration_import(integration):
+    """Run the integration import process with progress tracking.
+
+    :param QualysTotalCloudJSONLIntegration integration: Initialized integration object
+    """
+    assets_iterator, _ = integration.fetch_assets_and_findings()
+    logger.info("Syncing assets to RegScale...")
+
+    # Set up progress reporting and import data
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        TextColumn("•"),
+        TimeElapsedColumn(),
+        console=None,
+    ) as progress:
+        # Set up progress tasks
+        asset_task, finding_task, _, _ = _setup_progress_tracking(integration, progress)
+
+        # Import assets and findings
+        _import_assets(integration, assets_iterator, progress, asset_task)
+        _import_findings(integration, progress, finding_task)
+
+
+@click.command(name="import_total_cloud_xml")
+@regscale_ssp_id()
+@click.option(
+    "--xml_file",
+    "-f",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+    required=True,
+    help="Path to Qualys Total Cloud XML file to process.",
+)
+def import_total_cloud_from_xml(regscale_ssp_id: int, xml_file: str):
+    """
+    Import Qualys Total Cloud Assets and Vulnerabilities from an existing XML file using JSONL scanner.
+
+    This command processes an existing XML file instead of making an API call, useful for testing.
+    """
+    try:
+        logger.info(f"Processing Qualys Total Cloud XML file: {xml_file}")
+
+        # Parse the XML file
+        with open(xml_file, "r") as f:
+            xml_content = f.read()
+            response_data = xmltodict.parse(xml_content)
+
+        # Initialize the JSONLScannerIntegration implementation
+        integration = QualysTotalCloudJSONLIntegration(
+            plan_id=regscale_ssp_id, xml_data=response_data, file_path=xml_file
+        )
+
+        # Process data and generate JSONL files
+        if not _process_xml_to_jsonl(integration):
+            return
+
+        # Count and validate items for progress tracking
+        asset_count, finding_count = _count_jsonl_items(integration)
+        if asset_count == 0 and finding_count == 0:
+            logger.error("No assets or findings found in the processed data")
+            return
+
+        # Set up progress tracking and import to RegScale
+        _import_to_regscale(integration, asset_count, finding_count)
+
+        logger.info("Qualys Total Cloud XML file imported successfully with JSONL scanner.")
+    except Exception:
+        error_message = traceback.format_exc()
+        logger.error("Error occurred while processing Qualys XML file with JSONL scanner")
+        logger.error(error_message)
+
+
+def _process_xml_to_jsonl(integration):
+    """Process XML data and generate JSONL files."""
+    try:
+        logger.info("Fetching assets and findings...")
+        integration.fetch_assets_and_findings()
+
+        # Validate JSONL files
+        for file_path, file_type in [(integration.ASSETS_FILE, "Assets"), (integration.FINDINGS_FILE, "Findings")]:
+            if not os.path.exists(file_path) or os.path.getsize(file_path) == 0:
+                logger.error(f"{file_type} file not created or is empty: {file_path}")
+                return False
+
+        logger.info("Successfully created JSONL files.")
+        logger.info(f"Assets file size: {os.path.getsize(integration.ASSETS_FILE)} bytes")
+        logger.info(f"Findings file size: {os.path.getsize(integration.FINDINGS_FILE)} bytes")
+        return True
+    except Exception as e:
+        logger.error(f"Error fetching assets and findings: {str(e)}")
+        logger.error(traceback.format_exc())
+        return False
+
+
+def _count_jsonl_items(integration):
+    """Count assets and findings in JSONL files."""
+    # Count assets
+    assets_count = 0
+    asset_ids = []
+    try:
+        assets_iterator = integration._yield_items_from_jsonl(integration.ASSETS_FILE, IntegrationAsset)
+        for asset in assets_iterator:
+            assets_count += 1
+            if asset and hasattr(asset, "identifier"):
+                asset_ids.append(asset.identifier)
+
+        logger.debug(f"Asset IDs: {', '.join(asset_ids[:5])}{', ...' if len(asset_ids) > 5 else ''}")
+    except Exception as e:
+        logger.error(f"Error counting assets: {str(e)}")
+
+    # Count findings
+    findings_count = 0
+    try:
+        findings_iterator = integration._yield_items_from_jsonl(integration.FINDINGS_FILE, IntegrationFinding)
+        findings_count = sum(1 for _ in findings_iterator)
+    except Exception as e:
+        logger.error(f"Error counting findings: {str(e)}")
+
+    logger.info(f"Found {assets_count} assets and {findings_count} findings in JSONL files")
+    return assets_count, findings_count
+
+
+def _create_progress_bar():
+    """Create a progress bar that doesn't reset at 100%."""
+    from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn
+
+    class NonResettingProgress(Progress):
+        def update(self, task_id, **fields):
+            """Update a task."""
+            task = self._tasks[task_id]
+            completed_old = task.completed
+
+            # Update the task with the provided fields
+            for field_name, value in fields.items():
+                if field_name == "completed" and value is True:
+                    if task.total is not None:
+                        task.completed = task.total
+                else:
+                    setattr(task, field_name, value)
+
+            # Prevent task from being reset to 0% after reaching 100%
+            if completed_old == task.total and task.completed < task.total:
+                task.completed = task.total
+
+            self.refresh()
+
+    return NonResettingProgress(
+        SpinnerColumn(),
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        TextColumn("•"),
+        TimeElapsedColumn(),
+        console=None,
+    )
+
+
+def _track_progress_generator(iterator, progress_bar, task_id, id_attribute=None):
+    """Create a generator that tracks progress as items are yielded."""
+    processed = 0
+    item_ids = []
+
+    for item in iterator:
+        processed += 1
+
+        if id_attribute and hasattr(item, id_attribute):
+            item_ids.append(getattr(item, id_attribute))
+
+        progress_bar.update(task_id, completed=processed)
+        yield item
+
+    # Log debugging information
+    item_type = "items"
+    if id_attribute == "identifier":
+        item_type = "assets"
+    elif id_attribute == "external_id":
+        item_type = "findings"
+
+    logger.debug(f"Yielded {processed} {item_type} to RegScale")
+
+    if processed == 0:
+        logger.error(f"WARNING: No {item_type} were yielded to RegScale!")
+    elif item_ids:
+        logger.debug(f"First 10 {item_type} IDs: {item_ids[:10]}")
+
+
+def _import_to_regscale(integration, asset_count, finding_count):
+    """Import assets and findings to RegScale with progress tracking."""
+    progress = _create_progress_bar()
+
+    with progress:
+        # Create tasks
+        asset_task = progress.add_task(f"[green]Importing {asset_count} assets to RegScale...", total=asset_count)
+        finding_task = progress.add_task(
+            f"[yellow]Importing {finding_count} findings to RegScale...", visible=False, total=finding_count
+        )
+
+        # Import assets
+        try:
+            assets_iterator = integration._yield_items_from_jsonl(integration.ASSETS_FILE, IntegrationAsset)
+            tracked_assets = _track_progress_generator(assets_iterator, progress, asset_task, "identifier")
+            assets_imported = integration.update_regscale_assets(tracked_assets)
+            logger.info(f"Imported {assets_imported} assets to RegScale")
+            # Ensure the progress shows complete
+            progress.update(asset_task, completed=asset_count)
+        except Exception as e:
+            logger.error(f"Error importing assets to RegScale: {str(e)}")
+            logger.error(traceback.format_exc())
+            # Mark the task as completed even if there was an error
+            progress.update(asset_task, completed=asset_count)
+
+        # Import findings
+        progress.update(finding_task, visible=True)
+        try:
+            findings_iterator = integration._yield_items_from_jsonl(integration.FINDINGS_FILE, IntegrationFinding)
+            tracked_findings = _track_progress_generator(findings_iterator, progress, finding_task, "external_id")
+            findings_imported = integration.update_regscale_findings(tracked_findings)
+            logger.info(f"Imported {findings_imported} findings to RegScale")
+            # Ensure progress shows complete
+            progress.update(finding_task, completed=finding_count)
+        except Exception as e:
+            logger.error(f"Error importing findings to RegScale: {str(e)}")
+            logger.error(traceback.format_exc())
+            # Mark as completed even if there was an error
+            progress.update(finding_task, completed=finding_count)
+
+
 @qualys.command(name="export_scans")
 @save_output_to()
 @click.option(
@@ -172,6 +799,54 @@ def import_qualys_scans(
     )
 
 
+@qualys.command(name="import_policy_scans")
+@FlatFileImporter.common_scanner_options(
+    message="File path to the folder containing policy .csv files to process to RegScale.",
+    prompt="File path for Qualys files",
+    import_name="qualys_policy_scan",
+)
+@click.option(
+    "--skip_rows",
+    type=click.INT,
+    help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
+    default=5,
+)
+def import_policy_scans(
+    folder_path: os.PathLike[str],
+    regscale_ssp_id: int,
+    scan_date: datetime,
+    mappings_path: Path,
+    disable_mapping: bool,
+    skip_rows: int,
+    s3_bucket: str,
+    s3_prefix: str,
+    aws_profile: str,
+    upload_file: bool,
+):
+    """
+    Import Qualys policy scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
+    """
+    process_files_with_importer(
+        folder_path=str(folder_path),
+        importer_class=QualysPolicyScansImporter,
+        regscale_ssp_id=regscale_ssp_id,
+        importer_args={
+            "plan_id": regscale_ssp_id,
+            "name": "QualysPolicyScan",
+            "parent_id": regscale_ssp_id,
+            "parent_module": "securityplans",
+            "scan_date": scan_date,
+        },
+        mappings_path=str(mappings_path),
+        disable_mapping=disable_mapping,
+        skip_rows=skip_rows,
+        s3_bucket=s3_bucket,
+        s3_prefix=s3_prefix,
+        aws_profile=aws_profile,
+        upload_file=upload_file,
+    )
+
+
 @qualys.command(name="save_results")
 @save_output_to()
 @click.option(
@@ -348,80 +1023,6 @@ def import_was_scans(
     )
 
 
-@qualys.command(name="import_total_cloud")
-@regscale_ssp_id()
-@click.option(
-    "--include_tags",
-    "-t",
-    type=click.STRING,
-    required=False,
-    default=None,
-    help="Include tags in the import comma seperated string of tag names or ids, defaults to None.",
-)
-@click.option(
-    "--exclude_tags",
-    "-e",
-    type=click.STRING,
-    required=False,
-    default=None,
-    help="Exclude tags in the import comma seperated string of tag names or ids, defaults to None.",
-)
-def import_total_cloud_assets_and_vulnerabilities(regscale_ssp_id: int, include_tags: str, exclude_tags: str):
-    """
-    Import Qualys Total Cloud Assets and Vulnerabilities into RegScale via API."""
-    import_total_cloud_data_from_qualys_api(
-        security_plan_id=regscale_ssp_id, include_tags=include_tags, exclude_tags=exclude_tags
-    )
-
-
-@qualys.command(name="import_policy_scans")
-@FlatFileImporter.common_scanner_options(
-    message="File path to the folder containing policy .csv files to process to RegScale.",
-    prompt="File path for Qualys files",
-    import_name="qualys_policy_scan",
-)
-@click.option(
-    "--skip_rows",
-    type=click.INT,
-    help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
-    default=5,
-)
-def import_policy_scans(
-    folder_path: os.PathLike[str],
-    regscale_ssp_id: int,
-    scan_date: datetime,
-    mappings_path: Path,
-    disable_mapping: bool,
-    skip_rows: int,
-    s3_bucket: str,
-    s3_prefix: str,
-    aws_profile: str,
-    upload_file: bool,
-):
-    """
-    Import Qualys policy scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
-    """
-    process_files_with_importer(
-        folder_path=str(folder_path),
-        importer_class=QualysPolicyScansImporter,
-        regscale_ssp_id=regscale_ssp_id,
-        importer_args={
-            "plan_id": regscale_ssp_id,
-            "name": "QualysPolicyScan",
-            "parent_id": regscale_ssp_id,
-            "parent_module": "securityplans",
-            "scan_date": scan_date,
-        },
-        mappings_path=str(mappings_path),
-        disable_mapping=disable_mapping,
-        skip_rows=skip_rows,
-        s3_bucket=s3_bucket,
-        s3_prefix=s3_prefix,
-        aws_profile=aws_profile,
-        upload_file=upload_file,
-    )
-
-
 def process_files_with_importer(
     regscale_ssp_id: int,
     folder_path: str,
@@ -731,12 +1332,12 @@ def _get_qualys_api():
 def import_total_cloud_data_from_qualys_api(security_plan_id: int, include_tags: str, exclude_tags: str):
     """
     Function to get the total cloud data from Qualys API
+
     :param int security_plan_id: The ID of the plan to get the data for
     :param str include_tags: The tags to include in the data
     :param str exclude_tags: The tags to exclude from the data
     """
     try:
-
         qualys_url, QUALYS_API = _get_qualys_api()
         params = {
             "action": "list",
@@ -749,19 +1350,44 @@ def import_total_cloud_data_from_qualys_api(security_plan_id: int, include_tags:
                 params["tag_set_exclude"] = exclude_tags
             if include_tags:
                 params["tag_set_include"] = include_tags
+
         logger.info("Fetching Qualys Total Cloud data...")
         response = QUALYS_API.get(
             url=urljoin(qualys_url, "/api/2.0/fo/asset/host/vm/detection/"),
             headers=HEADERS,
             params=params,
         )
+
         if response and response.ok:
             logger.info("Total cloud data fetched. processing...")
             response_data = xmltodict.parse(response.text)
-            qt = QualysTotalCloudIntegration(plan_id=security_plan_id, xml_data=response_data)
-            qt.fetch_assets()
-            qt.fetch_findings()
 
+            # Create artifacts directory if it doesn't exist
+            os.makedirs("./artifacts", exist_ok=True)
+
+            # Write the XML data to a temporary file
+            temp_xml_file = "./artifacts/qualys_temp_data.xml"
+            with open(temp_xml_file, "w") as f:
+                f.write(response.text)
+
+            logger.info(f"Saved Qualys response data to {temp_xml_file}")
+
+            # Initialize the JSONLScannerIntegration implementation with the file path
+            integration = QualysTotalCloudJSONLIntegration(
+                plan_id=security_plan_id, xml_data=response_data, file_path=temp_xml_file
+            )
+
+            # Process assets and findings in one pass for efficiency
+            assets_iterator, findings_iterator = integration.fetch_assets_and_findings()
+
+            # Update RegScale with the processed data
+            logger.info("Syncing assets to RegScale...")
+            integration.update_regscale_assets(assets_iterator)
+
+            logger.info("Syncing findings to RegScale...")
+            integration.update_regscale_findings(findings_iterator)
+
+            logger.info("Qualys Total Cloud data imported successfully.")
         else:
             logger.error(
                 f"Received unexpected response from Qualys API: {response.status_code}: {response.text if response.text else 'response is null'}"
@@ -1451,3 +2077,28 @@ def get_asset_groups_from_qualys() -> list:
                 f"Unable to retrieve asset groups from Qualys.\nReceived: #{response.status_code}: {response.text}"
             )
     return asset_groups
+
+
+__all__ = [
+    "QualysTotalCloudJSONLIntegration",
+    "import_total_cloud",
+    "import_total_cloud_from_xml",
+    "save_results",
+    "sync_qualys",
+    "get_asset_groups",
+    "import_container_scans",
+    "import_was_scans",
+    "import_policy_scans",
+]
+
+# Register commands with the qualys group
+qualys.add_command(import_total_cloud)
+qualys.add_command(import_total_cloud_from_xml)
+qualys.add_command(export_past_scans)
+qualys.add_command(import_scans)
+qualys.add_command(import_policy_scans)
+qualys.add_command(save_results)
+qualys.add_command(sync_qualys)
+qualys.add_command(get_asset_groups)
+qualys.add_command(import_container_scans)
+qualys.add_command(import_was_scans)