cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/airbyte/util.py

@@ -0,0 +1,122 @@
+import logging
+import time
+from typing import Any
+
+import requests
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+class AirbyteClient:
+    """A client for interacting with the Airbyte API.
+    This client handles authentication and provides methods to make GET requests to the Airbyte API.
+    It automatically handles pagination for GET requests that return multiple pages of data.
+    """
+
+    def __init__(self, base_url: str, client_id: str, client_secret: str) -> None:
+        self._client_id = client_id
+        self._client_secret = client_secret
+        self.base_url = base_url
+        self._access_token_expiry: int | None = None
+        self._session = requests.Session()
+
+    def get(self, uri: str, params: dict | None = None, offset: int = 0) -> list[dict]:
+        """Make a GET request to the Airbyte API.
+        This method handles authentication and pagination.
+        Args:
+            uri (str): The URI to make the GET request to.
+            params (dict | None): Optional parameters to include in the request.
+            offset (int): The offset for pagination, defaults to 0.
+        Returns:
+            list[dict]: A list of dictionaries containing the data from the response.
+        """
+        self.authenticate()
+        if params is None:
+            params_with_pagination = {}
+        else:
+            params_with_pagination = params.copy()
+        params_with_pagination["offset"] = offset
+        response = self._session.get(
+            f"{self.base_url}{uri}", params=params_with_pagination, timeout=_TIMEOUT
+        )
+        response.raise_for_status()
+        data = response.json().get("data")
+        if response.json().get("next", "") != "":
+            data.extend(
+                self.get(
+                    uri,
+                    params=params,
+                    offset=offset + len(data),
+                )
+            )
+        return data
+
+    def authenticate(self) -> None:
+        """Authenticate with the Airbyte API using client credentials.
+        This method checks if the access token is still valid and renews it if necessary.
+        If the access token is expired or not set, it will make a request to obtain a new access token.
+        """
+        if self._access_token_expiry and self._access_token_expiry >= time.time():
+            return
+        self._session.headers.pop("Authorization", None)
+        payload = {
+            "grant-type": "client_credentials",
+            "client_id": self._client_id,
+            "client_secret": self._client_secret,
+        }
+        response = self._session.post(
+            f"{self.base_url}/applications/token", json=payload, timeout=_TIMEOUT
+        )
+        response.raise_for_status()
+        data = response.json()
+        self._session.headers["Authorization"] = (
+            f"Bearer {data.get('access_token', '')}"
+        )
+        token_expiry = data.get("expires_in", 0)
+        self._access_token_expiry = time.time() + token_expiry
+        logger.debug("Access token renewed, expires in %s seconds.", token_expiry)
+
+
+def normalize_airbyte_config(config: dict[str, Any]) -> dict[str, Any]:
+    """Normalize the Airbyte configuration dictionary.
+    This function takes a configuration dictionary and normalizes it by mapping keys to a standard set of keys.
+    This is useful for ensuring consistency across different configurations, and will allow to connect to existing nodes.
+    Args:
+        config (dict[str, Any]): The configuration dictionary to normalize.
+    Returns:
+        dict[str, Any]: A normalized configuration dictionary with standardized keys.
+    """
+    normalized_config = {}
+    for key in config:
+        if key in ("host", "port", "name", "region", "endpoint", "account", ""):
+            normalized_config[key] = config[key]
+        elif key in ("aws_region_name", "region_name", "s3_bucket_region"):
+            normalized_config["region"] = config[key]
+        elif key in ("queue_url", "url", "s3_endpoint"):
+            normalized_config["endpoint"] = config[key]
+        elif key in ("azure_blob_storage_account_name", "storage_account_name"):
+            normalized_config["account"] = config[key]
+        elif key in (
+            "azure_blob_storage_container_name",
+            "bucket",
+            "database",
+            "s3_bucket_name",
+        ):
+            normalized_config["name"] = config[key]
+    return normalized_config
+
+
+def list_to_string(lst: list[str]) -> str | None:
+    """Convert a list of strings to a comma-separated string."""
+    if len(lst) == 0:
+        return None
+    # Sublist
+    formated_list: list[str] = []
+    for item in lst:
+        if isinstance(item, list):
+            formated_list.append("|".join(item))
+        else:
+            formated_list.append(str(item))
+    return ",".join(formated_list)

cartography/intel/airbyte/workspaces.py

@@ -0,0 +1,63 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.airbyte.util import AirbyteClient
+from cartography.models.airbyte.workspace import AirbyteWorkspaceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: AirbyteClient,
+    org_id: str,
+    common_job_parameters: Dict[str, Any],
+) -> List[Dict[str, Any]]:
+    workspaces = get(api_session, org_id)
+    load_workspaces(
+        neo4j_session, workspaces, org_id, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return workspaces
+
+
+@timeit
+def get(
+    api_session: AirbyteClient,
+    org_id: str,
+) -> List[Dict[str, Any]]:
+    return api_session.get("/workspaces", params={"organizationId": org_id})
+
+
+@timeit
+def load_workspaces(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Airbyte Workspaces into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AirbyteWorkspaceSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AirbyteWorkspaceSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/aws/__init__.py

@@ -6,6 +6,7 @@ from typing import Dict
 from typing import Iterable
 from typing import List
 
+import aioboto3
 import boto3
 import botocore.exceptions
 import neo4j
@@ -49,12 +50,13 @@ def _build_aws_sync_kwargs(
 
 def _sync_one_account(
     neo4j_session: neo4j.Session,
-    boto3_session: boto3.session.Session,
+    boto3_session: boto3.Session,
     current_aws_account_id: str,
     update_tag: int,
     common_job_parameters: Dict[str, Any],
     regions: list[str] | None = None,
     aws_requested_syncs: Iterable[str] = RESOURCE_FUNCTIONS.keys(),
+    aioboto3_session: aioboto3.Session = aioboto3.Session(),
 ) -> None:
     # Autodiscover the regions supported by the account unless the user has specified the regions to sync.
     if not regions:
@@ -72,13 +74,20 @@ def _sync_one_account(
     for func_name in aws_requested_syncs:
         if func_name in RESOURCE_FUNCTIONS:
             # Skip permission relationships and tags for now because they rely on data already being in the graph
-            if func_name not in [
-                "permission_relationships",
-                "resourcegroupstaggingapi",
-            ]:
-                RESOURCE_FUNCTIONS[func_name](**sync_args)
-            else:
+            if func_name == "ecr:image_layers":
+                # has a different signature than the other functions (aioboto3_session replaces boto3_session)
+                RESOURCE_FUNCTIONS[func_name](
+                    neo4j_session,
+                    aioboto3_session,
+                    regions,
+                    current_aws_account_id,
+                    update_tag,
+                    common_job_parameters,
+                )
+            elif func_name in ["permission_relationships", "resourcegroupstaggingapi"]:
                 continue
+            else:
+                RESOURCE_FUNCTIONS[func_name](**sync_args)
         else:
             raise ValueError(
                 f'AWS sync function "{func_name}" was specified but does not exist. Did you misspell it?',
@@ -115,7 +124,7 @@ def _sync_one_account(
 
 
 def _autodiscover_account_regions(
-    boto3_session: boto3.session.Session,
+    boto3_session: boto3.Session,
     account_id: str,
 ) -> List[str]:
     regions: List[str] = []
@@ -136,7 +145,7 @@ def _autodiscover_account_regions(
 
 def _autodiscover_accounts(
     neo4j_session: neo4j.Session,
-    boto3_session: boto3.session.Session,
+    boto3_session: boto3.Session,
     account_id: str,
     sync_tag: int,
     common_job_parameters: Dict,
@@ -197,8 +206,10 @@ def _sync_multiple_accounts(
         if num_accounts == 1:
             # Use the default boto3 session because boto3 gets confused if you give it a profile name with 1 account
             boto3_session = boto3.Session()
+            aioboto3_session = aioboto3.Session()
         else:
             boto3_session = boto3.Session(profile_name=profile_name)
+            aioboto3_session = aioboto3.Session(profile_name=profile_name)
 
         _autodiscover_accounts(
             neo4j_session,
@@ -217,6 +228,7 @@ def _sync_multiple_accounts(
                 common_job_parameters,
                 regions=regions,
                 aws_requested_syncs=aws_requested_syncs,  # Could be replaced later with per-account requested syncs
+                aioboto3_session=aioboto3_session,
             )
         except Exception as e:
             if aws_best_effort_mode:
@@ -310,6 +322,9 @@ def start_aws_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
         "permission_relationships_file": config.permission_relationships_file,
+        "aws_guardduty_severity_threshold": config.aws_guardduty_severity_threshold,
+        "aws_cloudtrail_management_events_lookback_hours": config.aws_cloudtrail_management_events_lookback_hours,
+        "experimental_aws_inspector_batch": config.experimental_aws_inspector_batch,
     }
     try:
         boto3_session = boto3.Session()

cartography/intel/aws/acm.py

@@ -0,0 +1,124 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.acm.certificate import ACMCertificateSchema
+from cartography.stats import get_stats_client
+from cartography.util import aws_handle_regions
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_acm_certificates(
+    boto3_session: boto3.session.Session, region: str
+) -> list[dict[str, Any]]:
+    client = boto3_session.client("acm", region_name=region)
+    paginator = client.get_paginator("list_certificates")
+    summaries: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        summaries.extend(page.get("CertificateSummaryList", []))
+
+    details: list[dict[str, Any]] = []
+    for summary in summaries:
+        arn = summary["CertificateArn"]
+        resp = client.describe_certificate(CertificateArn=arn)
+        details.append(resp["Certificate"])
+    return details
+
+
+def transform_acm_certificates(
+    certificates: list[dict[str, Any]], region: str
+) -> list[dict[str, Any]]:
+    transformed: list[dict[str, Any]] = []
+    for cert in certificates:
+        item: dict[str, Any] = {
+            "Arn": cert["CertificateArn"],
+            "DomainName": cert.get("DomainName"),
+            "Type": cert.get("Type"),
+            "Status": cert.get("Status"),
+            "KeyAlgorithm": cert.get("KeyAlgorithm"),
+            "SignatureAlgorithm": cert.get("SignatureAlgorithm"),
+            "NotBefore": cert.get("NotBefore"),
+            "NotAfter": cert.get("NotAfter"),
+            "InUseBy": cert.get("InUseBy", []),
+            "Region": region,
+        }
+        # Extract ELBV2 Listener ARNs for relationship creation
+        listener_arns = [a for a in item["InUseBy"] if ":listener/" in a]
+        if listener_arns:
+            item["ELBV2ListenerArns"] = listener_arns
+        transformed.append(item)
+    return transformed
+
+
+@timeit
+def load_acm_certificates(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} ACM certificates for region {region} into graph.")
+    load(
+        neo4j_session,
+        ACMCertificateSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup_acm_certificates(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    logger.debug("Running ACM certificate cleanup job.")
+    GraphJob.from_node_schema(ACMCertificateSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing ACM certificates for region {region} in account {current_aws_account_id}."
+        )
+        certs = get_acm_certificates(boto3_session, region)
+        transformed = transform_acm_certificates(certs, region)
+        load_acm_certificates(
+            neo4j_session,
+            transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup_acm_certificates(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="ACMCertificate",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )