cartography 0.104.0rc2__py3-none-any.whl → 0.105.0__py3-none-any.whl

cartography/intel/entra/groups.py

@@ -0,0 +1,151 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.identity import ClientSecretCredential
+from msgraph import GraphServiceClient
+from msgraph.generated.models.directory_object import DirectoryObject
+from msgraph.generated.models.group import Group
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.entra.users import load_tenant
+from cartography.models.entra.group import EntraGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+async def get_entra_groups(client: GraphServiceClient) -> list[Group]:
+    """Get all groups from Microsoft Graph API with pagination."""
+    all_groups: list[Group] = []
+
+    request_configuration = client.groups.GroupsRequestBuilderGetRequestConfiguration(
+        query_parameters=client.groups.GroupsRequestBuilderGetQueryParameters(top=999)
+    )
+    page = await client.groups.get(request_configuration=request_configuration)
+    while page:
+        if page.value:
+            all_groups.extend(page.value)
+        if not page.odata_next_link:
+            break
+        page = await client.groups.with_url(page.odata_next_link).get()
+
+    return all_groups
+
+
+@timeit
+async def get_group_members(
+    client: GraphServiceClient, group_id: str
+) -> tuple[list[str], list[str]]:
+    """Get member user IDs and subgroup IDs for a given group."""
+    user_ids: list[str] = []
+    group_ids: list[str] = []
+    request_builder = client.groups.by_group_id(group_id).members
+    page = await request_builder.get()
+    while page:
+        if page.value:
+            for obj in page.value:
+                if isinstance(obj, DirectoryObject):
+                    odata_type = getattr(obj, "odata_type", "")
+                    if odata_type == "#microsoft.graph.user":
+                        user_ids.append(obj.id)
+                    elif odata_type == "#microsoft.graph.group":
+                        group_ids.append(obj.id)
+        if not page.odata_next_link:
+            break
+        page = await request_builder.with_url(page.odata_next_link).get()
+    return user_ids, group_ids
+
+
+def transform_groups(
+    groups: list[Group],
+    user_member_map: dict[str, list[str]],
+    group_member_map: dict[str, list[str]],
+) -> list[dict[str, Any]]:
+    """Transform API responses into dictionaries for ingestion."""
+    result: list[dict[str, Any]] = []
+    for g in groups:
+        transformed = {
+            "id": g.id,
+            "display_name": g.display_name,
+            "description": g.description,
+            "mail": g.mail,
+            "mail_nickname": g.mail_nickname,
+            "mail_enabled": g.mail_enabled,
+            "security_enabled": g.security_enabled,
+            "group_types": g.group_types,
+            "visibility": g.visibility,
+            "is_assignable_to_role": g.is_assignable_to_role,
+            "created_date_time": g.created_date_time,
+            "deleted_date_time": g.deleted_date_time,
+            "member_ids": user_member_map.get(g.id, []),
+            "member_group_ids": group_member_map.get(g.id, []),
+        }
+        result.append(transformed)
+    return result
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    groups: list[dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    logger.info(f"Loading {len(groups)} Entra groups")
+    load(
+        neo4j_session,
+        EntraGroupSchema(),
+        groups,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def cleanup_groups(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(EntraGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+async def sync_entra_groups(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """Sync Entra groups."""
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
+    )
+    client = GraphServiceClient(
+        credential, scopes=["https://graph.microsoft.com/.default"]
+    )
+
+    groups = await get_entra_groups(client)
+
+    user_member_map: dict[str, list[str]] = {}
+    group_member_map: dict[str, list[str]] = {}
+    for group in groups:
+        try:
+            users, subgroups = await get_group_members(client, group.id)
+            user_member_map[group.id] = users
+            group_member_map[group.id] = subgroups
+        except Exception as e:
+            logger.error(f"Failed to fetch members for group {group.id}: {e}")
+            user_member_map[group.id] = []
+            group_member_map[group.id] = []
+
+    transformed_groups = transform_groups(groups, user_member_map, group_member_map)
+
+    load_tenant(neo4j_session, {"id": tenant_id}, update_tag)
+    load_groups(neo4j_session, transformed_groups, update_tag, tenant_id)
+    cleanup_groups(neo4j_session, common_job_parameters)

cartography/intel/entra/ou.py

@@ -22,12 +22,28 @@ async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
     Get all OUs from Microsoft Graph API with pagination support
     """
     all_units: list[AdministrativeUnit] = []
-    request = client.directory.administrative_units.request()
 
-    while request:
-        response = await request.get()
-        all_units.extend(response.value)
-        request = response.odata_next_link if response.odata_next_link else None
+    # Initialize first page request
+    current_request = client.directory.administrative_units
+
+    while current_request:
+        try:
+            response = await current_request.get()
+            if response and response.value:
+                all_units.extend(response.value)
+
+                # Handle next page using OData link
+                if response.odata_next_link:
+                    current_request = client.directory.administrative_units.with_url(
+                        response.odata_next_link
+                    )
+                else:
+                    current_request = None
+            else:
+                current_request = None
+        except Exception as e:
+            logger.error(f"Failed to retrieve administrative units: {str(e)}")
+            current_request = None
 
     return all_units
 

cartography/intel/trivy/__init__.py

@@ -0,0 +1,161 @@
+import logging
+from typing import Any
+
+import boto3
+from neo4j import Session
+
+from cartography.client.aws import list_accounts
+from cartography.client.aws.ecr import get_ecr_images
+from cartography.config import Config
+from cartography.intel.trivy.scanner import cleanup
+from cartography.intel.trivy.scanner import get_json_files_in_s3
+from cartography.intel.trivy.scanner import sync_single_image_from_s3
+from cartography.stats import get_stats_client
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client("trivy.scanner")
+
+
+@timeit
+def get_scan_targets(
+    neo4j_session: Session,
+    account_ids: list[str] | None = None,
+) -> set[str]:
+    """
+    Return list of ECR images from all accounts in the graph.
+    """
+    if not account_ids:
+        aws_accounts = list_accounts(neo4j_session)
+    else:
+        aws_accounts = account_ids
+
+    ecr_images: set[str] = set()
+    for account_id in aws_accounts:
+        for _, _, image_uri, _, _ in get_ecr_images(neo4j_session, account_id):
+            ecr_images.add(image_uri)
+
+    return ecr_images
+
+
+def _get_intersection(
+    images_in_graph: set[str], json_files: set[str], trivy_s3_prefix: str
+) -> list[tuple[str, str]]:
+    """
+    Get the intersection of ECR images in the graph and S3 scan results.
+
+    Args:
+        images_in_graph: Set of ECR images in the graph
+        json_files: Set of S3 object keys for JSON files
+        trivy_s3_prefix: S3 prefix path containing scan results
+
+    Returns:
+        List of tuples (image_uri, s3_object_key)
+    """
+    intersection = []
+    prefix_len = len(trivy_s3_prefix)
+    for s3_object_key in json_files:
+        # Sample key "123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
+        # Sample key "folder/derp/123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
+        # Remove the prefix and the .json suffix
+        image_uri = s3_object_key[prefix_len:-5]
+
+        if image_uri in images_in_graph:
+            intersection.append((image_uri, s3_object_key))
+
+    return intersection
+
+
+@timeit
+def sync_trivy_aws_ecr_from_s3(
+    neo4j_session: Session,
+    trivy_s3_bucket: str,
+    trivy_s3_prefix: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+    boto3_session: boto3.Session,
+) -> None:
+    """
+    Sync Trivy scan results from S3 for AWS ECR images.
+
+    Args:
+        neo4j_session: Neo4j session for database operations
+        trivy_s3_bucket: S3 bucket containing scan results
+        trivy_s3_prefix: S3 prefix path containing scan results
+        update_tag: Update tag for tracking
+        common_job_parameters: Common job parameters for cleanup
+        boto3_session: boto3 session for S3 operations
+    """
+    logger.info(
+        f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
+    )
+
+    images_in_graph: set[str] = get_scan_targets(neo4j_session)
+    json_files: set[str] = get_json_files_in_s3(
+        trivy_s3_bucket, trivy_s3_prefix, boto3_session
+    )
+    intersection: list[tuple[str, str]] = _get_intersection(
+        images_in_graph, json_files, trivy_s3_prefix
+    )
+
+    if len(intersection) == 0:
+        logger.error(
+            f"Trivy sync was configured, but there are no ECR images with S3 json scan results in bucket "
+            f"'{trivy_s3_bucket}' with prefix '{trivy_s3_prefix}'. "
+            "Skipping Trivy sync to avoid potential data loss. "
+            "Please check the S3 bucket and prefix configuration. We expect the json files in s3 to be named "
+            f"`<image_uri>.json` and to be in the same bucket and prefix as the scan results. If the prefix is "
+            "a folder, it MUST end with a trailing slash '/'. "
+        )
+        logger.error(f"JSON files in S3: {json_files}")
+        raise ValueError("No ECR images with S3 json scan results found.")
+
+    logger.info(f"Processing {len(intersection)} ECR images with S3 scan results")
+    for image_uri, s3_object_key in intersection:
+        sync_single_image_from_s3(
+            neo4j_session,
+            image_uri,
+            update_tag,
+            trivy_s3_bucket,
+            s3_object_key,
+            boto3_session,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
+    """
+    Start Trivy scan ingestion from S3.
+
+    Args:
+        neo4j_session: Neo4j session for database operations
+        config: Configuration object containing S3 settings
+    """
+    # Check if S3 configuration is provided
+    if not config.trivy_s3_bucket:
+        logger.info("Trivy S3 configuration not provided. Skipping Trivy ingestion.")
+        return
+
+    # Default to empty string if s3 prefix is not provided
+    if config.trivy_s3_prefix is None:
+        config.trivy_s3_prefix = ""
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    # Get ECR images to scan
+    boto3_session = boto3.Session()
+
+    sync_trivy_aws_ecr_from_s3(
+        neo4j_session,
+        config.trivy_s3_bucket,
+        config.trivy_s3_prefix,
+        config.update_tag,
+        common_job_parameters,
+        boto3_session,
+    )
+
+    # Support other Trivy resource types here e.g. if Google Cloud has images.