regscale-cli 6.20.3.1__py3-none-any.whl → 6.20.4.1__py3-none-any.whl

regscale/__init__.py

@@ -1 +1 @@
-__version__ = "6.20.3.1"
+__version__ = "6.20.4.1"

regscale/integrations/commercial/__init__.py

@@ -308,6 +308,7 @@ show_mapping(prisma, "prisma", "csv")
         "get_asset_groups": "regscale.integrations.commercial.qualys.__init__.get_asset_groups",
         "import_total_cloud_xml": "regscale.integrations.commercial.qualys.__init__.import_total_cloud_from_xml",
         "import_total_cloud": "regscale.integrations.commercial.qualys.__init__.import_total_cloud",
+        "validate_csv": "regscale.integrations.commercial.qualys.__init__.validate_csv",
     },
     name="qualys",
 )

regscale/integrations/commercial/jira.py

@@ -460,7 +460,7 @@ def create_regscale_task_from_jira(config: dict, jira_issue: jiraIssue, parent_i
     :rtype: Task
     """
     description = jira_issue.fields.description
-    due_date = jira_issue.fields.duedate
+    due_date = map_jira_due_date(jira_issue, config)
     status = convert_task_status(jira_issue.fields.status.name)
     status_change_date = convert_datetime_to_regscale_string(
         datetime.strptime(jira_issue.fields.statuscategorychangedate, "%Y-%m-%dT%H:%M:%S.%f%z")
@@ -468,9 +468,6 @@ def create_regscale_task_from_jira(config: dict, jira_issue: jiraIssue, parent_i
     title = jira_issue.fields.summary
     date_closed = None
     percent_complete = None
-    if not due_date:
-        delta = config["issues"]["jira"]["medium"]
-        due_date = convert_datetime_to_regscale_string(datetime.now() + timedelta(days=delta))
     if status == "Closed":
         date_closed = status_change_date
         percent_complete = 100
@@ -534,23 +531,45 @@ def process_tasks_for_sync(
     insert_tasks = []
     update_tasks = []
     close_tasks = []
+
+    # Create a map of existing tasks by their Jira key for easier lookup
+    # Only include tasks that have an otherIdentifier
+    existing_task_map = {
+        task.otherIdentifier: task
+        for task in existing_tasks
+        if hasattr(task, "otherIdentifier") and task.otherIdentifier
+    }
+
     for jira_issue in jira_issues:
-        task = create_regscale_task_from_jira(config, jira_issue, parent_id, parent_module)
-        existing_task = next((t for t in existing_tasks if t == task and not t.extra_data.get("updated", False)), None)
+        # Create a RegScale task from the Jira issue
+        jira_task = create_regscale_task_from_jira(config, jira_issue, parent_id, parent_module)
+
+        # Check if we have a matching task in RegScale
+        existing_task = existing_task_map.get(jira_issue.key)
+
         if existing_task:
-            task.id = existing_task.id
-            if (jira_issue.fields.status.name.lower() == "done" and existing_task.status not in closed_statuses) or (
-                task.status in closed_statuses and task != existing_task
+            # Check if task is closed in Jira and needs to be closed in regscale
+            if jira_task.status in closed_statuses and existing_task.status not in closed_statuses:
+                existing_task.status = "Closed"
+                existing_task.percentComplete = 100
+                existing_task.dateClosed = safe_datetime_str(jira_issue.fields.statuscategorychangedate)
+                close_tasks.append(existing_task)
+
+            # Check if update needed
+            elif (
+                jira_task.title != existing_task.title
+                or jira_task.description != existing_task.description
+                or jira_task.status != existing_task.status
+                or (jira_task.dueDate != existing_task.dueDate and jira_issue.fields.duedate)
             ):
-                task.status = "Closed"
-                task.percentComplete = 100
-                task.dateClosed = safe_datetime_str(jira_issue.fields.statuscategorychangedate)
-                close_tasks.append(task)
-            elif task != existing_task:
-                update_tasks.append(task)
+                jira_task.id = existing_task.id  # Preserve the RegScale ID
+                update_tasks.append(jira_task)
         else:
-            insert_tasks.append(task)
+            # Task only exists in Jira - needs to be created in RegScale
+            insert_tasks.append(jira_task)
+
         progress.update(progress_task, advance=1)
+
     return insert_tasks, update_tasks, close_tasks
 
 

regscale/integrations/commercial/qualys/__init__.py

@@ -28,6 +28,7 @@ from regscale.core.app.utils.app_utils import (
     create_progress_object,
 )
 from regscale.core.app.utils.file_utils import download_from_s3
+from regscale.integrations.commercial.qualys.qualys_error_handler import QualysErrorHandler
 from regscale.integrations.commercial.qualys.scanner import QualysTotalCloudJSONLIntegration
 from regscale.integrations.commercial.qualys.variables import QualysVariables
 from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
@@ -411,26 +412,52 @@ def _fetch_qualys_api_data(include_tags, exclude_tags):
     :param str exclude_tags: Tags to exclude from the query
     :return: Parsed XML data or None if request failed
     """
+    from regscale.integrations.commercial.qualys.qualys_error_handler import QualysErrorHandler
+
     qualys_url, qualys_api = _get_qualys_api()
     params = _prepare_qualys_params(include_tags, exclude_tags)
 
     logger.info("Fetching Qualys Total Cloud data with JSONL scanner...")
-    response = qualys_api.get(
-        url=urljoin(qualys_url, "/api/2.0/fo/asset/host/vm/detection/"),
-        headers=HEADERS,
-        params=params,
-        verify=ScannerVariables.sslVerify,  # Apply SSL verification setting
-    )
 
-    if not response or not response.ok:
-        logger.error(
-            f"Received unexpected response from Qualys API: {response.status_code}: {response.text if response.text else 'response is null'}"
+    try:
+        response = qualys_api.get(
+            url=urljoin(qualys_url, "/api/2.0/fo/asset/host/vm/detection/"),
+            headers=HEADERS,
+            params=params,
+            verify=ScannerVariables.sslVerify,  # Apply SSL verification setting
         )
-        return None
 
-    # Process API response
-    logger.info("Total cloud data fetched. Processing with JSONL scanner...")
-    return xmltodict.parse(response.text)
+        # Use the error handler to validate the response
+        is_valid, error_message, parsed_data = QualysErrorHandler.validate_response(response)
+
+        if not is_valid:
+            logger.error(f"Qualys API request failed: {error_message}")
+
+            # If we have parsed data, extract detailed error information
+            if parsed_data:
+                error_details = QualysErrorHandler.extract_error_details(parsed_data)
+                QualysErrorHandler.log_error_details(error_details)
+
+                # Check if this is a retryable error
+                error_code = error_details.get("error_code")
+                if error_code and QualysErrorHandler.should_retry(error_code):
+                    retry_after = error_details.get("retry_after", 60)
+                    logger.warning(f"This error may be retryable. Consider retrying after {retry_after} seconds.")
+
+                # Check if this is a fatal error that should stop processing
+                if error_code and QualysErrorHandler.is_fatal_error(error_code):
+                    logger.error("Fatal error encountered. Please check your credentials and permissions.")
+
+            return None
+
+        # Process API response
+        logger.info("Total cloud data fetched successfully. Processing with JSONL scanner...")
+        return parsed_data
+
+    except Exception as e:
+        logger.error(f"Unexpected error during Qualys API request: {e}")
+        logger.debug(traceback.format_exc())
+        return None
 
 
 def _initialize_integration(regscale_ssp_id, response_data, vulnerability_creation, ssl_verify):
@@ -501,13 +528,34 @@ def import_total_cloud_from_xml(regscale_ssp_id: int, xml_file: str):
 
     This command processes an existing XML file instead of making an API call, useful for testing.
     """
+    from regscale.integrations.commercial.qualys.qualys_error_handler import QualysErrorHandler
+
     try:
         logger.info(f"Processing Qualys Total Cloud XML file: {xml_file}")
 
-        # Parse the XML file
+        # Parse the XML file with error handling
         with open(xml_file, "r") as f:
             xml_content = f.read()
-            response_data = xmltodict.parse(xml_content)
+
+        # Use the error handler to safely parse XML
+        success, response_data, error_message = QualysErrorHandler.parse_xml_safely(xml_content)
+
+        if not success:
+            logger.error(f"Failed to parse XML file: {error_message}")
+
+            # If we have partial data, try to extract error details
+            if response_data:
+                error_details = QualysErrorHandler.extract_error_details(response_data)
+                QualysErrorHandler.log_error_details(error_details)
+
+            return
+
+        # Check for Qualys-specific errors in the parsed data
+        error_details = QualysErrorHandler.extract_error_details(response_data)
+        if error_details.get("has_error"):
+            logger.error("XML file contains Qualys error response")
+            QualysErrorHandler.log_error_details(error_details)
+            return
 
         # Initialize the JSONLScannerIntegration implementation
         integration = QualysTotalCloudJSONLIntegration(
@@ -740,7 +788,25 @@ def import_scans(
     aws_profile: str,
     upload_file: bool,
 ):
-    """Import .csv scans from Qualys"""
+    """
+    Import vulnerability scans from Qualys CSV files.
+
+    This command processes Qualys CSV export files and imports assets and vulnerabilities
+    into RegScale. The CSV files must contain specific required headers.
+
+    TROUBLESHOOTING:
+    If you encounter "No columns to parse from file" errors, try:
+    1. Run 'regscale qualys validate_csv -f <file_path>' first
+    2. Adjust the --skip_rows parameter (default: 129)
+    3. Check that your CSV file has the required headers
+
+    REQUIRED HEADERS:
+    Severity, Title, Exploitability, CVE ID, Solution, DNS, IP,
+    QG Host ID, OS, NetBIOS, FQDN
+
+    For detailed format requirements, see the documentation at:
+    regscale/integrations/commercial/qualys/QUALYS_CSV_FORMAT.md
+    """
     import_qualys_scans(
         folder_path=folder_path,
         regscale_ssp_id=regscale_ssp_id,
@@ -1612,7 +1678,7 @@ def update_and_insert_assets(
                 qualysId=qualys_asset["ASSET_ID"],
             )
             # avoid duplication
-            if r_asset.qualysId not in set(v["qualysId"] for v in insert_assets):
+            if r_asset.qualysId not in {v["qualysId"] for v in insert_assets}:
                 insert_assets.append(r_asset)
         try:
             created_assets = Asset.batch_create(insert_assets, job_progress)
@@ -1901,19 +1967,18 @@ def map_issue_data_to_assets(assets: list[dict], qualys_issue_data: dict) -> lis
 
 def lookup_asset(asset_list: list, asset_id: str = None) -> list[Asset]:
     """
-    Function to look up an asset in the asset list and returns an Asset object
+    Look up assets in the asset list by Qualys asset ID or return all unique assets.
+
+    Args:
+        asset_list (list): List of assets from RegScale.
+        asset_id (str, optional): Qualys asset ID to search for. Defaults to None.
 
-    :param list asset_list: List of assets from RegScale
-    :param str asset_id: Qualys asset ID to search for, defaults to None
-    :return: list of Asset objects
-    :rtype: list[Asset]
+    Returns:
+        list[Asset]: List of unique Asset objects matching the asset_id, or all unique assets if asset_id is None.
     """
     if asset_id:
-        results = [asset for asset in asset_list if getattr(asset, "qualysId", None) == asset_id]
-    else:
-        results = [asset for asset in asset_list]
-    # Return unique list
-    return list(set(results)) or []
+        return list({asset for asset in asset_list if getattr(asset, "qualysId", None) == asset_id})
+    return list(set(asset_list)) or []
 
 
 def map_qualys_severity_to_regscale(severity: int) -> tuple[str, str]:
@@ -2045,7 +2110,7 @@ def inner_join(reg_list: list, qualys_list: list) -> list:
     :rtype: list
     """
 
-    set1 = set(getattr(lst, "qualysId", None) for lst in reg_list)
+    set1 = {getattr(asset, "qualysId", None) for asset in reg_list}
     data = []
     try:
         data = [list_qualys for list_qualys in qualys_list if getattr(list_qualys, "ASSET_ID", None) in set1]
@@ -2079,6 +2144,211 @@ def get_asset_groups_from_qualys() -> list:
     return asset_groups
 
 
+def _get_required_headers() -> list[str]:
+    """
+    Get the list of required headers for Qualys CSV validation.
+
+    :return: List of required header names
+    :rtype: list[str]
+    """
+    return [
+        "Severity",
+        "Title",
+        "Exploitability",
+        "CVE ID",
+        "Solution",
+        "DNS",
+        "IP",
+        "QG Host ID",
+        "OS",
+        "NetBIOS",
+        "FQDN",
+    ]
+
+
+def _read_csv_file(file_path: str, skip_rows: int, console):
+    """
+    Read and validate CSV file structure.
+
+    :param str file_path: Path to the CSV file
+    :param int skip_rows: Number of rows to skip
+    :param console: Rich console instance for output
+    :return: DataFrame if successful, None if failed
+    :rtype: Optional[pd.DataFrame]
+    """
+    import pandas as pd
+
+    console.print(f"[blue]Reading CSV file: {file_path}[/blue]")
+    console.print(f"[blue]Skipping first {skip_rows} rows[/blue]")
+
+    try:
+        if skip_rows > 0:
+            df = pd.read_csv(file_path, skiprows=skip_rows - 1, on_bad_lines="warn")
+        else:
+            df = pd.read_csv(file_path, on_bad_lines="warn")
+
+        if df.empty:
+            console.print("[red]❌ File is empty after skipping rows[/red]")
+            return None
+
+        if len(df.columns) == 0:
+            console.print("[red]❌ No columns found in the file[/red]")
+            return None
+
+        console.print("[green]✅ Successfully read CSV file[/green]")
+        console.print(f"[green]✅ Found {len(df.columns)} columns and {len(df)} rows[/green]")
+        return df
+
+    except pd.errors.EmptyDataError:
+        console.print("[red]❌ File is empty or contains no parseable data[/red]")
+        console.print("[yellow]💡 Try adjusting the skip_rows parameter[/yellow]")
+        return None
+    except pd.errors.ParserError as e:
+        console.print(f"[red]❌ Error parsing CSV file: {e}[/red]")
+        console.print("[yellow]💡 Check if the file is properly formatted CSV[/yellow]")
+        return None
+
+
+def _display_columns_table(df, console):
+    """
+    Display a table showing all columns found in the CSV.
+
+    :param df: DataFrame containing the CSV data
+    :param console: Rich console instance for output
+    """
+    from rich.table import Table
+
+    table = Table(title="Columns Found in CSV")
+    table.add_column("Index", style="cyan")
+    table.add_column("Column Name", style="magenta")
+
+    for i, col in enumerate(df.columns):
+        table.add_row(str(i), str(col))
+
+    console.print(table)
+
+
+def _check_required_headers(df, console):
+    """
+    Check for required headers and display results.
+
+    :param df: DataFrame containing the CSV data
+    :param console: Rich console instance for output
+    :return: Tuple of (missing_headers, found_headers)
+    :rtype: tuple[list[str], list[str]]
+    """
+    required_headers = _get_required_headers()
+
+    console.print("\n[blue]Checking for required headers:[/blue]")
+    missing_headers = []
+    found_headers = []
+
+    for header in required_headers:
+        if header in df.columns:
+            found_headers.append(header)
+            console.print(f"[green]✅ {header}[/green]")
+        else:
+            missing_headers.append(header)
+            console.print(f"[red]❌ {header}[/red]")
+
+    return missing_headers, found_headers
+
+
+def _display_header_validation_summary(missing_headers, console):
+    """
+    Display summary of header validation results.
+
+    :param list[str] missing_headers: List of missing required headers
+    :param console: Rich console instance for output
+    """
+    if missing_headers:
+        console.print(f"\n[yellow]⚠️  Missing {len(missing_headers)} required headers[/yellow]")
+        console.print("[yellow]You may need to adjust the skip_rows parameter or check your file format[/yellow]")
+    else:
+        console.print("\n[green]🎉 All required headers found! File should import successfully.[/green]")
+
+
+def _display_sample_data(df, console):
+    """
+    Display sample data from the CSV file.
+
+    :param df: DataFrame containing the CSV data
+    :param console: Rich console instance for output
+    """
+    from rich.table import Table
+
+    if len(df) == 0:
+        return
+
+    console.print("\n[blue]Sample data (first 3 rows):[/blue]")
+    sample_table = Table()
+
+    # Add columns (limit to first 5 for readability)
+    display_cols = list(df.columns)[:5]
+    for col in display_cols:
+        sample_table.add_column(str(col)[:20], style="cyan")
+
+    # Add rows
+    for i in range(min(3, len(df))):
+        row_data = [str(df.iloc[i][col])[:30] for col in display_cols]
+        sample_table.add_row(*row_data)
+
+    console.print(sample_table)
+
+    if len(df.columns) > 5:
+        console.print(f"[dim]... and {len(df.columns) - 5} more columns[/dim]")
+
+
+@qualys.command(name="validate_csv")
+@click.option(
+    "--file_path",
+    "-f",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+    required=True,
+    help="Path to the Qualys CSV file to validate.",
+    prompt="Enter the path to the Qualys CSV file",
+)
+@click.option(
+    "--skip_rows",
+    type=click.INT,
+    help="The number of rows in the file to skip to get to the column headers, defaults to 129.",
+    default=129,
+)
+def validate_csv(file_path: str, skip_rows: int):
+    """
+    Validate a Qualys CSV file format and headers before importing.
+
+    This command helps troubleshoot CSV file issues by checking:
+    - File readability
+    - Proper column headers after skipping rows
+    - Required headers presence
+    """
+    from rich.console import Console
+
+    console = Console()
+
+    try:
+        # Read and validate CSV file
+        df = _read_csv_file(file_path, skip_rows, console)
+        if df is None:
+            return
+
+        # Display columns found
+        _display_columns_table(df, console)
+
+        # Check required headers
+        missing_headers, _ = _check_required_headers(df, console)
+
+        # Display validation summary
+        _display_header_validation_summary(missing_headers, console)
+
+        # Show sample data
+        _display_sample_data(df, console)
+
+    except Exception as e:
+        console.print(f"[red]❌ Unexpected error: {e}[/red]")
+
+
 __all__ = [
     "QualysTotalCloudJSONLIntegration",
     "import_total_cloud",