regscale-cli 6.20.10.0__py3-none-any.whl → 6.21.1.0__py3-none-any.whl

regscale/_version.py

@@ -33,7 +33,7 @@ def get_version_from_pyproject() -> str:
                     return match.group(1)
     except Exception:
         pass
-    return "6.20.10.0"  # fallback version
+    return "6.21.1.0"  # fallback version
 
 
 __version__ = get_version_from_pyproject()

regscale/core/app/application.py

@@ -245,6 +245,7 @@ class Application(metaclass=Singleton):
             "snowPassword": "<snowPassword>",
             "snowUrl": "<mySnowUrl>",
             "snowUserName": "<snowUserName>",
+            "sonarUrl": "https://sonarcloud.io",
             "sonarToken": "<mySonarToken>",
             "tenableAccessKey": "<tenableAccessKeyGoesHere>",
             "tenableSecretKey": "<tenableSecretKeyGoesHere>",
@@ -295,7 +296,10 @@ class Application(metaclass=Singleton):
         self.running_in_airflow = os.getenv("REGSCALE_AIRFLOW") == "true"
         if isinstance(config, str):
             config = self._read_config_from_str(config)
-        self.config = self._gen_config(config)
+        if self.running_in_airflow:
+            self.config = self._fetch_config_from_regscale(config)
+        else:
+            self.config = self._gen_config(config)
         self.os = platform.system()
         self.input_host = ""
         # Ensure maxThreads is an integer for ThreadManager
@@ -536,10 +540,6 @@ class Application(metaclass=Singleton):
             self.logger.debug("Successfully retrieved config from Click context.")
             return click_config
 
-        if self.running_in_airflow:
-            if airflow_config := self._get_airflow_config(config):
-                self.logger.debug("Successfully retrieved config from Airflow.")
-                return airflow_config
         try:
             if config and self.local_config:
                 self.logger.debug(f"Config provided as :\n{type(config)}")
@@ -572,6 +572,13 @@ class Application(metaclass=Singleton):
         return config
 
     def _get_airflow_config(self, config: Optional[Union[dict, str]] = None) -> Optional[dict]:
+        """
+        Get config from Airflow DAG config, or from the environment variables if not provided.
+
+        :param Optional[Union[dict, str]] config: Configuration dictionary, defaults to None
+        :return: Configuration dictionary
+        :rtype: Optional[dict]
+        """
         if config:
             self.logger.debug(f"Received config from Airflow as: {type(config)}")
             # check to see if config is a string because airflow can pass a string instead of a dict

regscale/core/app/internal/set_permissions.py

@@ -1,11 +1,9 @@
 import click
 from pathlib import Path
 import os
-from openpyxl import Workbook, load_workbook
-from openpyxl.styles import Protection, Font, NamedStyle
-from openpyxl.worksheet.worksheet import Worksheet
+from rich.progress import track
+from openpyxl import Workbook
 from openpyxl.worksheet.datavalidation import DataValidation
-from openpyxl import Workbook, load_workbook
 from openpyxl.utils.dataframe import dataframe_to_rows
 
 from regscale.core.app.logz import create_logger
@@ -118,38 +116,71 @@ def import_permissions_workbook(file: Path):
     """
     # Read in the spreadsheet
     records = get_records(file=file)
-    for record in records:
+    for index in track(
+        range(len(records)),
+        description="Processing rbacs...",
+    ):
+        record = records[index]
+        # Check the records:
+
         if not Group.get_object(record["group_id"]):
             logger.error(f"Group {record['group_id']} doesn't exist in this instance.  Skipping row")
             continue
 
-        regscale_module_id = Modules.get_module_to_id(record["regscale_module"])
-        regscale_id = record["regscale_id"]
-        if record[READ_UPDATE] == "R":
-            permissions = 1
-        elif record[READ_UPDATE] == "RU":
-            permissions = 2
-        else:
-            permissions = 0
-
-        # Set the record to private
         my_class = Modules.module_to_class(record["regscale_module"])
-        obj = my_class.get_object(regscale_id)
+        obj = my_class.get_object(record["regscale_id"])
         if not obj:
-            logger.error(f"RegScale {record['regscale_module']} record {regscale_id} doesn't exist. Skipping row")
+            logger.error(
+                f"RegScale {record['regscale_module']} record {record['regscale_id']} doesn't exist. Skipping row"
+            )
             continue
 
-        # Add the permission
-        RBAC.add(
-            module_id=regscale_module_id,
-            parent_id=regscale_id,
-            group_id=record["group_id"],
-            permission_type=permissions,
+        # Set the permissions
+        set_permissions(record=record)
+
+
+def set_permissions(record: dict):
+    """
+    Sets the permissions for each record
+
+    :param dict record: permissions dictionary
+    """
+
+    if record[READ_UPDATE] == "R":
+        permissions = 1
+    elif record[READ_UPDATE] == "RU":
+        permissions = 2
+    else:
+        permissions = 0
+
+    # Add the permission
+    if not RBAC.add(
+        module_id=Modules.get_module_to_id(record["regscale_module"]),
+        parent_id=record["regscale_id"],
+        group_id=record["group_id"],
+        permission_type=permissions,
+    ):
+        logger.warning(
+            f"Failed to set permissions for {record['regscale_module']} {record['regscale_id']} for group {record['group_id']}"
+        )
+        return
+
+    if not RBAC.public(
+        module_id=Modules.get_module_to_id(record["regscale_module"]),
+        parent_id=record["regscale_id"],
+        is_public=0 if record[PUBLIC_PRIVATE] == "private" else 1,
+    ):
+        logger.warning(
+            f"Failed to set public/private for {record['regscale_module']} {record['regscale_id']} for group {record['group_id']}"
         )
-        RBAC.public(
-            module_id=regscale_module_id,
-            parent_id=regscale_id,
-            is_public=0 if record[PUBLIC_PRIVATE] == "private" else 1,
+        return
+
+    if not RBAC.reset(
+        module_id=Modules.get_module_to_id(record["regscale_module"]),
+        parent_id=record["regscale_id"],
+    ):
+        logger.warning(
+            f"Failed to proliferate permissions for {record['regscale_module']} {record['regscale_id']} for group {record['group_id']}"
         )
 
 

regscale/integrations/commercial/__init__.py

@@ -43,9 +43,8 @@ show_mapping(aqua, "aqua")
     cls=LazyGroup,
     lazy_subcommands={
         "sync_assets": "regscale.integrations.commercial.aws.cli.sync_assets",
-        "sync_findings": "regscale.integrations.commercial.aws.cli.sync_findings",
+        "sync_findings_and_assets": "regscale.integrations.commercial.aws.cli.sync_findings_and_assets",
         "inspector": "regscale.integrations.commercial.aws.cli.inspector",
-        "inventory": "regscale.integrations.commercial.aws.cli.inventory",
     },
     name="aws",
 )

regscale/integrations/commercial/amazon/common.py

@@ -11,6 +11,8 @@ from dateutil import parser
 
 from regscale.core.app.utils.app_utils import create_logger
 
+logger = create_logger()
+
 
 def check_finding_severity(comment: Optional[str]) -> str:
     """Check the severity of the finding
@@ -93,6 +95,63 @@ def get_comments(finding: dict) -> str:
 
 
 def fetch_aws_findings(aws_client: BaseClient) -> list:
+    """Fetch AWS Findings with optimized rate limiting and pagination
+
+    :param BaseClient aws_client: AWS Security Hub Client
+    :return: AWS Findings
+    :rtype: list
+    """
+    findings = []
+    try:
+        # Use optimized SecurityHubPuller for better performance
+        from regscale.integrations.commercial.aws.security_hub import SecurityHubPuller
+
+        # Extract credentials from the client to create SecurityHubPuller
+        session = aws_client._client_config.__dict__.get("_user_provided_options", {})
+        region = session.meta.region_name
+
+        # Create SecurityHubPuller with same credentials as the client
+        puller = SecurityHubPuller(region_name=region)
+
+        # Use existing client instead of creating new one to maintain credentials
+        puller.client = aws_client
+
+        # Fetch all findings with optimized pagination and rate limiting
+        logger.info("Using optimized SecurityHubPuller for findings retrieval...")
+        findings = puller.get_all_findings_with_retries()
+
+        logger.info(f"Successfully fetched {len(findings)} findings with rate limiting")
+
+    except ImportError:
+        # Fallback to original method if SecurityHubPuller not available
+        logger.warning("SecurityHubPuller not available, falling back to basic client")
+        findings = fallback_fetch_aws_findings(aws_client)
+    except ClientError as cex:
+        logger.error("Unexpected error: %s", cex)
+    except Exception as e:
+        logger.error("Error using SecurityHubPuller, falling back to basic client: %s", e)
+        findings = fallback_fetch_aws_findings(aws_client)
+
+    return findings
+
+
+def fallback_fetch_aws_findings(aws_client: BaseClient) -> list:
+    """Fallback method to fetch AWS Findings without pagination
+
+    :param BaseClient aws_client: AWS Security Hub Client
+    :return: AWS Findings
+    :rtype: list
+    """
+    findings = []
+    try:
+        response = aws_client.get_findings()
+        findings = response.get("Findings", [])
+    except ClientError as cex:
+        create_logger().error("Unexpected error when fetching resources from AWS: %s", cex)
+    return findings
+
+
+def fetch_aws_findings_v2(aws_client: BaseClient) -> list:
     """Fetch AWS Findings
 
     :param BaseClient aws_client: AWS Security Hub Client
@@ -101,7 +160,25 @@ def fetch_aws_findings(aws_client: BaseClient) -> list:
     """
     findings = []
     try:
-        findings = aws_client.get_findings()["Findings"]
+        response = aws_client.get_findings_v2()
+        findings = response.get("Findings", [])
     except ClientError as cex:
-        create_logger().error("Unexpected error: %s", cex)
+        create_logger().error("Unexpected error when fetching resources from AWS: %s", cex)
     return findings
+
+
+def fetch_aws_resources(aws_client: BaseClient) -> list:
+    """Fetch AWS Resources
+
+    :param BaseClient aws_client: AWS Security Hub Client
+    :return: AWS Resources
+    :rtype: list
+    """
+    resources = []
+    try:
+        response = aws_client.get_resources_v2()
+        resources = response.get("Resources", [])
+        logger.info(f"Fetched {len(resources)} resources from Security Hub")
+    except ClientError as cex:
+        create_logger().error("Unexpected error when fetching resources from AWS: %s", cex)
+    return resources

regscale/integrations/commercial/aws/cli.py

@@ -265,13 +265,13 @@ def import_aws_scans(
     "--region",
     type=str,
     default=os.environ.get("AWS_REGION", "us-east-1"),
-    help="AWS region to collect inventory from. Default is us-east-1.",
+    help="AWS region to collect findings from",
 )
 @click.option(
     "--regscale_id",
     "--id",
     type=click.INT,
-    help="RegScale will create and update assets as children of this record.",
+    help="RegScale will create and update findings as children of this record.",
     required=True,
 )
 @click.option(
@@ -286,7 +286,7 @@ def import_aws_scans(
     type=str,
     required=False,
     help="AWS secret access key",
-    envvar="AWS_SECRET_ACCESS_KEY",
+    default=os.getenv("AWS_SECRET_ACCESS_KEY"),
 )
 @click.option(
     "--aws_session_token",
@@ -302,21 +302,195 @@ def sync_findings(
     aws_secret_access_key: Optional[str] = None,
     aws_session_token: Optional[str] = None,
 ) -> None:
-    """Sync AWS Security Hub Findings."""
+    """
+    Sync AWS Security Hub findings to RegScale.
+
+    This command fetches findings from AWS Security Hub and creates/updates
+    corresponding issues in RegScale.
+    """
+    try:
+        logger.info("Starting AWS Security Hub findings sync to RegScale...")
+        from .scanner import AWSInventoryIntegration
+
+        scanner = AWSInventoryIntegration(plan_id=regscale_id)
+        findings_processed = scanner.sync_findings(
+            plan_id=regscale_id,
+            region=region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            aws_session_token=aws_session_token,
+        )
+        logger.info(f"AWS Security Hub findings sync completed successfully. Processed {findings_processed} findings.")
+    except Exception as e:
+        logger.error(f"Error syncing AWS Security Hub findings: {e}", exc_info=True)
+        raise click.ClickException(str(e))
+
+
+@awsv2.command(name="sync_findings_and_assets")
+@click.option(
+    "--region",
+    type=str,
+    default=os.environ.get("AWS_REGION", "us-east-1"),
+    help="AWS region to collect findings and assets from",
+)
+@click.option(
+    "--regscale_id",
+    "--id",
+    type=click.INT,
+    help="RegScale will create and update findings and assets as children of this record.",
+    required=True,
+)
+@click.option(
+    "--aws_access_key_id",
+    type=str,
+    required=False,
+    help="AWS access key ID",
+    envvar="AWS_ACCESS_KEY_ID",
+)
+@click.option(
+    "--aws_secret_access_key",
+    type=str,
+    required=False,
+    help="AWS secret access key",
+    default=os.getenv("AWS_SECRET_ACCESS_KEY"),
+)
+@click.option(
+    "--aws_session_token",
+    type=click.STRING,
+    required=False,
+    help="AWS Session ID",
+    default=os.environ.get("AWS_SESSION_TOKEN"),
+)
+def sync_findings_and_assets(
+    region: str,
+    regscale_id: int,
+    aws_access_key_id: Optional[str] = None,
+    aws_secret_access_key: Optional[str] = None,
+    aws_session_token: Optional[str] = None,
+) -> None:
+    """
+    Sync AWS Security Hub findings and automatically discovered assets to RegScale.
+
+    This command fetches findings from AWS Security Hub, creates/updates corresponding
+    issues in RegScale, and also creates assets for the resources referenced in the findings.
+    This provides a comprehensive view by creating both the security findings and the
+    underlying AWS resources they reference.
+    """
     try:
-        logger.info("Starting AWS findings sync to RegScale...")
+        logger.info("Starting AWS Security Hub findings and assets sync to RegScale...")
         from .scanner import AWSInventoryIntegration
 
         scanner = AWSInventoryIntegration(plan_id=regscale_id)
-        scanner.sync_findings(
+        findings_processed, assets_processed = scanner.sync_findings_and_assets(
             plan_id=regscale_id,
             region=region,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
             aws_session_token=aws_session_token,
         )
-        if not scanner.errors:
-            logger.info("AWS finding sync completed successfully.")
+        logger.info(
+            f"AWS Security Hub sync completed successfully. "
+            f"Processed {findings_processed} findings and {assets_processed} assets."
+        )
+    except Exception as e:
+        logger.error(f"Error syncing AWS Security Hub findings and assets: {e}", exc_info=True)
+        raise click.ClickException(str(e))
+
+
+@awsv2.group()
+def findings():
+    """AWS Security Hub findings commands."""
+    pass
+
+
+@findings.command(name="collect")
+@click.option(
+    "--region",
+    type=str,
+    default=os.getenv("AWS_REGION", "us-east-1"),
+    help="AWS region to collect findings from. Default is us-east-1.",
+)
+@click.option(
+    "--aws_access_key_id",
+    type=str,
+    required=False,
+    help="AWS access key ID",
+    envvar="AWS_ACCESS_KEY_ID",
+)
+@click.option(
+    "--aws_secret_access_key",
+    type=str,
+    required=False,
+    help="AWS secret access key",
+    envvar="AWS_SECRET_ACCESS_KEY",
+)
+@click.option(
+    "--aws_session_token",
+    type=click.STRING,
+    required=False,
+    help="AWS Session ID",
+    default=os.environ.get("AWS_SESSION_TOKEN"),
+)
+@click.option(
+    "--output",
+    type=click.Path(dir_okay=False, writable=True),
+    help="Output file path (JSON format). Default: artifacts/aws/findings.json",
+    required=False,
+)
+def collect_findings(
+    region: str,
+    aws_access_key_id: Optional[str],
+    aws_secret_access_key: Optional[str],
+    aws_session_token: Optional[str],
+    output: Optional[str],
+) -> None:
+    """
+    Collect AWS Security Hub findings.
+
+    This command fetches findings from AWS Security Hub and displays them to stdout
+    or saves them to a JSON file. The findings include security issues, compliance
+    violations, and other security-related information from AWS Security Hub.
+
+    If no output file is specified, findings will be saved to artifacts/aws/findings.json
+    by default. Use --output - to display to stdout instead.
+    """
+    try:
+        import boto3
+        from regscale.integrations.commercial.amazon.common import fetch_aws_findings
+        from regscale.models import DateTimeEncoder
+
+        logger.info("Collecting AWS Security Hub findings...")
+
+        # Create AWS session
+        session = boto3.Session(
+            region_name=region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            aws_session_token=aws_session_token,
+        )
+        client = session.client("securityhub")
+
+        # Fetch findings
+        findings = fetch_aws_findings(aws_client=client)
+
+        logger.info(f"AWS Security Hub findings collected successfully. Found {len(findings)} finding(s).")
+
+        # Default output path
+        if output is None:
+            output = os.path.join("artifacts", "aws", "findings.json")
+
+        if output == "-":
+            # Output to stdout
+            click.echo(json.dumps(findings, indent=2, cls=DateTimeEncoder))
+        else:
+            # Save to file
+            # Ensure the artifacts directory exists
+            os.makedirs(os.path.dirname(output), exist_ok=True)
+
+            with open(output, "w", encoding="utf-8") as f:
+                json.dump(findings, f, indent=2, cls=DateTimeEncoder)
+            logger.info(f"Findings saved to {output}")
+
     except Exception as e:
-        logger.error(f"Error syncing AWS finding(s): {e}", exc_info=True)
+        logger.error(f"Error collecting AWS Security Hub findings: {e}", exc_info=True)
         raise click.ClickException(str(e))