regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

regscale/core/login.py

@@ -6,6 +6,7 @@ from os import getenv
 from typing import Optional, Tuple
 from urllib.parse import urljoin
 
+from requests.exceptions import HTTPError
 from regscale.core.app.api import Api
 from regscale.core.app.utils.app_utils import error_and_exit
 
@@ -14,10 +15,11 @@ logger = logging.getLogger("regscale")
 
 def get_regscale_token(
     api: Api,
-    username: str = getenv("REGSCALE_USER"),
-    password: str = getenv("REGSCALE_PASSWORD"),
-    domain: str = getenv("REGSCALE_DOMAIN"),
+    username: Optional[str] = getenv("REGSCALE_USER"),
+    password: Optional[str] = getenv("REGSCALE_PASSWORD"),
+    domain: Optional[str] = getenv("REGSCALE_DOMAIN"),
     mfa_token: Optional[str] = "",
+    app_id: Optional[int] = 1,
 ) -> Tuple[str, str]:
     """
     Authenticate with RegScale and return a token
@@ -27,6 +29,7 @@ def get_regscale_token(
     :param str password: a string defaulting to the envar REGSCALE_PASSWORD
     :param str domain: a string representing the RegScale domain, checks environment REGSCALE_DOMAIN
     :param Optional[str] mfa_token: MFA token to login with
+    :param Optional[int] app_id: The app ID to login with
     :raises EnvironmentError: if domain is not passed or retrieved
     :return: a tuple of user_id and auth_token
     :rtype: Tuple[str, str]
@@ -47,7 +50,19 @@ def get_regscale_token(
     logger.info("Logging into: %s", domain)
     # suggest structuring the login paths so that they all exist in one place
     url = urljoin(domain, "/api/authentication/login")
-    response = api.post(url=url, json=auth, headers={})
+    try:
+        # Try to authenticate with the new API version
+        auth["appId"] = app_id
+        response = api.post(url=url, json=auth, headers={"X-Api-Version": "2.0"})
+        if response is None:
+            raise HTTPError("No response received from api.post(). Possible connection issue or internal error.")
+        response.raise_for_status()
+        app_id_compatible = True
+    except HTTPError:
+        # Fallback to the old API version
+        del auth["appId"]
+        response = api.post(url=url, json=auth, headers={})
+        app_id_compatible = False
     error_msg = "Unable to authenticate with RegScale. Please check your credentials."
     if response is None:
         logger.error("No response received from api.post(). Possible connection issue or internal error.")
@@ -63,4 +78,6 @@ def get_regscale_token(
         error_and_exit(f"{error_msg}\n{response.status_code}: {response.text}")
     if isinstance(response_dict, str):
         response_dict = json.loads(response_dict)
+    if app_id_compatible:
+        return response_dict["accessToken"]["id"], response_dict["accessToken"]["authToken"]
     return response_dict["id"], response_dict["auth_token"]

regscale/core/utils/async_graphql_client.py

@@ -0,0 +1,363 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Generic async GraphQL client for concurrent query processing in RegScale CLI."""
+
+import asyncio
+import logging
+from typing import Any, Callable, Dict, List, Optional
+
+import anyio
+from gql import Client, gql
+from gql.transport.aiohttp import AIOHTTPTransport
+from gql.transport.aiohttp import log as aiohttp_logger
+
+from regscale.core.app.application import Application
+
+logger = logging.getLogger("regscale")
+
+
+class GraphQLQueryError(Exception):
+    """Exception raised when a GraphQL query fails."""
+
+    pass
+
+
+class GraphQLAuthenticationError(GraphQLQueryError):
+    """Exception raised when GraphQL authentication fails."""
+
+    pass
+
+
+class AsyncRegScaleGraphQLClient:
+    """
+    Generic async GraphQL client optimized for concurrent RegScale API queries.
+
+    This client can execute multiple GraphQL queries concurrently, significantly
+    improving performance when fetching paginated data from RegScale.
+    """
+
+    def __init__(
+        self,
+        endpoint: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: float = 30.0,
+        max_concurrent: int = 5,
+        token_refresh_callback: Optional[Callable[[], str]] = None,
+    ):
+        """
+        Initialize the async GraphQL client.
+
+        :param str endpoint: GraphQL endpoint URL
+        :param Optional[Dict[str, str]] headers: HTTP headers for requests
+        :param float timeout: Request timeout in seconds
+        :param int max_concurrent: Maximum concurrent requests
+        :param Optional[Callable[[], str]] token_refresh_callback: Callback to refresh auth token
+        """
+        self.app = Application()
+        self.ssl_verify = self.app.config.get("sslVerify", True)
+        self.endpoint = endpoint
+        self.headers = headers or {"Authorization": self.app.config.get("token")}
+        self.timeout = timeout
+        self.max_concurrent = max_concurrent
+        self.token_refresh_callback = token_refresh_callback
+        self._semaphore = anyio.Semaphore(max_concurrent)
+
+        # Set logging level for aiohttp transport
+        aiohttp_logger.setLevel(logging.CRITICAL)
+
+    def _create_client(self) -> Client:
+        """
+        Create a new GQL client with transport.
+
+        Each concurrent request needs its own transport and client to avoid
+        "Transport is already connected" errors.
+
+        :return: New GQL Client instance
+        :rtype: Client
+        """
+
+        # Create the transport with authentication headers
+        # Note: AIOHTTPTransport uses ssl parameter, not verify_ssl
+        import ssl as ssl_module
+
+        ssl_context = None
+        if not self.ssl_verify:
+            # SECURITY WARNING: SSL verification is intentionally disabled
+            # This is required for environments with self-signed certificates or corporate proxies
+            # where SSL verification cannot be performed. This should only be used when:
+            # 1. Working with internal/trusted networks with self-signed certificates
+            # 2. Behind corporate proxies that intercept SSL/TLS traffic
+            # 3. Development/testing environments
+            # DO NOT use in production with untrusted networks
+            logger.warning(
+                "SSL certificate verification is disabled. "
+                "This is insecure and should only be used in controlled environments "
+                "with self-signed certificates or corporate proxies."
+            )
+            # Use default context but disable verification for compatibility
+            # The default context still uses secure TLS 1.2+ protocols
+            ssl_context = ssl_module.create_default_context()  # NOSONAR - Uses TLS 1.2+ by default in Python 3.9+
+            ssl_context.check_hostname = False  # NOSONAR - Intentionally disabled when sslVerify=false is configured
+            ssl_context.verify_mode = ssl_module.CERT_NONE  # NOSONAR - Required for self-signed certs/corporate proxies
+
+        transport = AIOHTTPTransport(
+            url=self.endpoint,
+            headers=self.headers,
+            timeout=int(self.timeout),
+            ssl=ssl_context if not self.ssl_verify else True,
+        )
+
+        # Create and return a new GQL client
+        return Client(
+            transport=transport,
+            fetch_schema_from_transport=False,  # Skip schema introspection for performance
+        )
+
+    def _is_auth_error(self, error_msg: str) -> bool:
+        """
+        Check if an error message indicates an authentication issue.
+
+        :param str error_msg: Error message to check
+        :return: True if the error is authentication-related
+        :rtype: bool
+        """
+        auth_indicators = ["AUTH_NOT_AUTHENTICATED", "UNAUTHENTICATED", "401", "403"]
+        return any(indicator in error_msg for indicator in auth_indicators)
+
+    async def _execute_single_attempt(
+        self, query: str, variables: Optional[Dict[str, Any]], progress_callback: Optional[Callable], task_name: str
+    ) -> Dict[str, Any]:
+        """
+        Execute a single query attempt.
+
+        :param str query: GraphQL query string
+        :param Optional[Dict[str, Any]] variables: Query variables
+        :param Optional[callable] progress_callback: Callback for progress updates
+        :param str task_name: Name for progress tracking
+        :return: Query response data
+        :rtype: Dict[str, Any]
+        """
+        if progress_callback:
+            progress_callback(task_name, "requesting")
+
+        # Parse the query string to a GraphQL document
+        doc = gql(query)
+
+        # Create a new client for this request to avoid connection conflicts
+        client = self._create_client()
+
+        # Execute the query using the GQL client
+        async with client as session:
+            result = await session.execute(doc, variable_values=variables or {})
+
+        if progress_callback:
+            progress_callback(task_name, "completed")
+
+        return result
+
+    async def execute_query(
+        self,
+        query: str,
+        variables: Optional[Dict[str, Any]] = None,
+        progress_callback: Optional[Callable] = None,
+        task_name: str = "GraphQL Query",
+    ) -> Dict[str, Any]:
+        """
+        Execute a single GraphQL query asynchronously.
+
+        :param str query: GraphQL query string
+        :param Optional[Dict[str, Any]] variables: Query variables
+        :param Optional[callable] progress_callback: Callback for progress updates
+        :param str task_name: Name for progress tracking
+        :return: Query response data
+        :rtype: Dict[str, Any]
+        """
+        async with self._semaphore:  # Limit concurrent requests
+            if progress_callback:
+                progress_callback(task_name, "starting")
+
+            logger.debug("Async GraphQL request to %s", self.endpoint)
+
+            # Try up to 2 times (initial + 1 retry with token refresh)
+            max_attempts = 2
+            for attempt in range(max_attempts):
+                try:
+                    result = await self._execute_single_attempt(query, variables, progress_callback, task_name)
+                    return result
+
+                except Exception as e:
+                    error_msg = str(e)
+
+                    # Check for authentication errors and retry if possible
+                    if self._is_auth_error(error_msg):
+                        if attempt < max_attempts - 1 and self.token_refresh_callback:
+                            logger.warning("Authentication error, refreshing token and retrying")
+                            new_token = self.token_refresh_callback()
+                            self.headers["Authorization"] = f"Bearer {new_token}"
+                            continue
+
+                    # Log and re-raise other errors
+                    error_msg = f"Error in {task_name}: {error_msg}"
+                    logger.error(error_msg)
+                    if progress_callback:
+                        progress_callback(task_name, "failed")
+                    raise GraphQLQueryError(error_msg) from e
+
+            # Should never reach here, but just in case
+            raise GraphQLQueryError(f"Failed to execute query after {max_attempts} attempts")
+
+    async def execute_paginated_query_concurrent(
+        self,
+        query_builder: Callable[[int, int], str],
+        topic_key: str,
+        total_count: int,
+        page_size: int = 50,
+        starting_skip: int = 0,
+        progress_callback: Optional[Callable] = None,
+        task_name: str = "Paginated Query",
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute a paginated GraphQL query with concurrent page fetching.
+
+        :param Callable[[int, int], str] query_builder: Function that builds query with skip and take
+        :param str topic_key: Key to extract nodes from response
+        :param int total_count: Total number of items expected
+        :param int page_size: Items per page (default: 50)
+        :param int starting_skip: Starting skip value (default: 0)
+        :param Optional[callable] progress_callback: Callback for progress updates
+        :param str task_name: Name for progress tracking
+        :return: All nodes from all pages
+        :rtype: List[Dict[str, Any]]
+        """
+        # Calculate number of pages needed
+        num_pages = (total_count + page_size - 1) // page_size
+
+        logger.debug("Fetching %d pages concurrently for %s", num_pages, task_name)
+
+        # Create tasks for all pages
+        tasks = []
+        for page_num in range(num_pages):
+            skip = starting_skip + (page_num * page_size)
+            query = query_builder(skip, page_size)
+
+            page_task_name = f"{task_name} (Page {page_num + 1}/{num_pages})"
+            tasks.append(
+                self._fetch_single_page(
+                    query=query,
+                    variables={},
+                    topic_key=topic_key,
+                    page_num=page_num + 1,
+                    progress_callback=progress_callback,
+                    task_name=page_task_name,
+                )
+            )
+
+        # Execute all page fetches concurrently
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Collect all nodes from successful pages
+        all_nodes: List[Dict[str, Any]] = []
+        for i, result in enumerate(results):
+            if isinstance(result, Exception):
+                logger.error("Error fetching page %d: %s", i + 1, str(result))
+            elif isinstance(result, list):
+                all_nodes.extend(result)
+
+        return all_nodes
+
+    async def _fetch_single_page(
+        self,
+        query: str,
+        variables: Dict[str, Any],
+        topic_key: str,
+        page_num: int,
+        progress_callback: Optional[Callable] = None,
+        task_name: str = "Page",
+    ) -> List[Dict[str, Any]]:
+        """
+        Fetch a single page of results.
+
+        :param str query: GraphQL query string
+        :param Dict[str, Any] variables: Query variables
+        :param str topic_key: Key to extract nodes from response
+        :param int page_num: Page number for logging
+        :param Optional[callable] progress_callback: Callback for progress updates
+        :param str task_name: Name for progress tracking
+        :return: Nodes from this page
+        :rtype: List[Dict[str, Any]]
+        """
+        try:
+            data = await self.execute_query(
+                query=query, variables=variables, progress_callback=progress_callback, task_name=task_name
+            )
+
+            topic_data = data.get(topic_key, {})
+            nodes = topic_data.get("items", [])
+
+            # Handle case where nodes is explicitly None
+            if nodes is None:
+                nodes = []
+
+            if progress_callback:
+                progress_callback(task_name, f"fetched_{len(nodes)}_items")
+
+            return nodes
+
+        except Exception as e:
+            logger.error("Error fetching page %d: %s", page_num, str(e))
+            raise
+
+
+def run_async_paginated_query(
+    endpoint: str,
+    headers: Dict[str, str],
+    query_builder: Callable[[int, int], str],
+    topic_key: str,
+    total_count: int,
+    page_size: int = 50,
+    starting_skip: int = 0,
+    max_concurrent: int = 5,
+    timeout: int = 60,
+    progress_callback: Optional[Callable] = None,
+    task_name: str = "Paginated Query",
+    token_refresh_callback: Optional[Callable[[], str]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Convenience function to run async paginated query from synchronous code.
+
+    :param str endpoint: GraphQL endpoint URL
+    :param Dict[str, str] headers: HTTP headers
+    :param Callable[[int, int], str] query_builder: Function to build query with skip and take
+    :param str topic_key: Key to extract data from response
+    :param int total_count: Total number of items to fetch
+    :param int page_size: Items per page
+    :param int starting_skip: Starting skip value
+    :param int max_concurrent: Maximum concurrent requests
+    :param int timeout: Request timeout in seconds
+    :param Optional[callable] progress_callback: Progress callback
+    :param str task_name: Task name for progress tracking
+    :param Optional[Callable[[], str]] token_refresh_callback: Callback to refresh auth token
+    :return: Query results
+    :rtype: List[Dict[str, Any]]
+    """
+
+    async def _run():
+        client = AsyncRegScaleGraphQLClient(
+            endpoint=endpoint,
+            headers=headers,
+            max_concurrent=max_concurrent,
+            timeout=timeout,
+            token_refresh_callback=token_refresh_callback,
+        )
+        return await client.execute_paginated_query_concurrent(
+            query_builder=query_builder,
+            topic_key=topic_key,
+            total_count=total_count,
+            page_size=page_size,
+            starting_skip=starting_skip,
+            progress_callback=progress_callback,
+            task_name=task_name,
+        )
+
+    # Use anyio.run for better compatibility
+    return anyio.run(_run)

regscale/core/utils/date.py

@@ -2,8 +2,10 @@
 # -*- coding: utf-8 -*-
 """Utility functions for handling date and datetime conversions"""
 
+import calendar
 import datetime
 import logging
+import re
 from typing import Any, List, Optional, Union
 
 
@@ -39,7 +41,8 @@ def date_str(date_object: Union[str, datetime.datetime, datetime.date, None], da
             return date_object.strftime(date_format)
 
         return date_object.isoformat()
-    except Exception:
+    except (AttributeError, TypeError, ValueError) as e:
+        logger.debug(f"Error converting date object to string: {e}")
         return ""
 
 
@@ -82,6 +85,7 @@ def date_obj(date_str: Union[str, datetime.datetime, datetime.date, int, None])
 def datetime_obj(date_str: Union[str, datetime.datetime, datetime.date, int, None]) -> Optional[datetime.datetime]:
     """
     Convert a string, datetime, date, integer, or timestamp string to a datetime object.
+    If the day of the month is invalid (e.g., November 31), adjusts to the last valid day of that month.
 
     :param Union[str, datetime.datetime, datetime.date, int, None] date_str: The value to convert.
     :return: The datetime object.
@@ -93,6 +97,10 @@ def datetime_obj(date_str: Union[str, datetime.datetime, datetime.date, int, Non
         try:
             return parse(date_str)
         except ParserError as e:
+            # Try to fix invalid day of month (e.g., 2023/11/31 -> 2023/11/30)
+            if fixed_date := _fix_invalid_day_of_month(date_str):
+                return fixed_date
+
             if date_str and str(date_str).lower() not in ["n/a", "none"]:
                 logger.warning(f"Warning could not parse date string: {date_str}\n{e}")
             return None
@@ -105,6 +113,74 @@ def datetime_obj(date_str: Union[str, datetime.datetime, datetime.date, int, Non
     return None
 
 
+def _parse_date_components(match_groups: tuple) -> tuple[int, int, int]:
+    """
+    Parse year, month, day from regex match groups.
+
+    :param tuple match_groups: Tuple of matched groups from regex
+    :return: Tuple of (year, month, day)
+    :rtype: tuple[int, int, int]
+    """
+    if len(match_groups[0]) == 4:  # First group is year (YYYY/MM/DD)
+        return int(match_groups[0]), int(match_groups[1]), int(match_groups[2])
+    # Last group is year (MM/DD/YYYY)
+    return int(match_groups[2]), int(match_groups[0]), int(match_groups[1])
+
+
+def _adjust_invalid_day(year: int, month: int, day: int) -> Optional[datetime.datetime]:
+    """
+    Adjust invalid day of month to last valid day.
+
+    :param int year: Year
+    :param int month: Month
+    :param int day: Day
+    :return: Adjusted datetime or None if invalid
+    :rtype: Optional[datetime.datetime]
+    """
+    last_valid_day = calendar.monthrange(year, month)[1]
+    if day <= last_valid_day:
+        return None
+
+    logger.warning(f"Invalid day {day} for month {month}/{year}. Adjusting to last valid day: {last_valid_day}")
+    try:
+        return datetime.datetime(year, month, last_valid_day)
+    except ValueError:
+        return None
+
+
+def _fix_invalid_day_of_month(date_str: str) -> Optional[datetime.datetime]:
+    """
+    Attempt to fix an invalid day of month in a date string.
+    For example, 2023/11/31 would become 2023/11/30.
+
+    :param str date_str: The date string to fix
+    :return: A datetime object with a valid day, or None if it can't be fixed
+    :rtype: Optional[datetime.datetime]
+    """
+    try:
+        patterns = [
+            r"(\d{4})[/-](\d{1,2})[/-](\d{1,2})",  # YYYY/MM/DD or YYYY-MM-DD
+            r"(\d{1,2})[/-](\d{1,2})[/-](\d{4})",  # MM/DD/YYYY or MM-DD-YYYY
+        ]
+
+        for pattern in patterns:
+            if not (match := re.search(pattern, date_str)):
+                continue
+
+            year, month, day = _parse_date_components(match.groups())
+
+            if not (1 <= month <= 12):
+                continue
+
+            if result := _adjust_invalid_day(year, month, day):
+                return result
+
+        return None
+    except (ValueError, TypeError, AttributeError, IndexError) as e:
+        logger.debug(f"Could not fix invalid day of month for: {date_str} - {e}")
+        return None
+
+
 def time_str(time_obj: Union[str, datetime.datetime, datetime.time]) -> str:
     """
     Convert a datetime/time object to a string.

regscale/dev/cli.py

@@ -2,8 +2,11 @@
 
 import contextlib
 import os
+import re
 import sys
 
+from pathlib import Path
+
 import click
 from rich.console import Console
 
@@ -231,5 +234,28 @@ def update_docs(readme_token: str, confluence_user: str, confluence_token: str,
                 update_confluence(api, confluence_url, root, file)
 
 
+@cli.command()
+@click.option("--version", "-v", type=click.STRING, help="The version to upgrade the CLI to use.")
+@click.option("--current", "-c", is_flag=True, help="Get the current version of the CLI.")
+def version(version: str, current: bool) -> None:
+    """Manage the version of the regscale-cli package."""
+    from regscale.dev.version import (
+        get_current_version,
+        update_fallback_version_in_version_py,
+        update_version_in_pyproject_toml,
+    )
+
+    if current:
+        print(get_current_version())
+        return
+
+    if not version:
+        print("❌ Please provide a version to upgrade to using the --version flag.")
+        return
+
+    update_version_in_pyproject_toml(version)
+    update_fallback_version_in_version_py(version)
+
+
 if __name__ == "__main__":
     cli()