cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/util.py

@@ -5,6 +5,7 @@ from functools import partial
 from functools import wraps
 from importlib.resources import open_binary
 from importlib.resources import read_text
+from itertools import islice
 from string import Template
 from typing import Any
 from typing import Awaitable
@@ -24,6 +25,8 @@ import backoff
 import boto3
 import botocore
 import neo4j
+from botocore.exceptions import EndpointConnectionError
+from botocore.parsers import ResponseParserError
 
 from cartography.graph.job import GraphJob
 from cartography.graph.statement import get_job_shortname
@@ -33,10 +36,27 @@ from cartography.stats import ScopedStatsClient
 logger = logging.getLogger(__name__)
 
 
+def is_service_control_policy_explicit_deny(
+    error: botocore.exceptions.ClientError,
+) -> bool:
+    """Return True if the ClientError was caused by an explicit service control policy deny."""
+    error_code = error.response.get("Error", {}).get("Code")
+    if error_code not in {"AccessDenied", "AccessDeniedException"}:
+        return False
+
+    message = error.response.get("Error", {}).get("Message")
+    if not message:
+        return False
+
+    lowered = message.lower()
+    return "explicit deny" in lowered and "service control policy" in lowered
+
+
 STATUS_SUCCESS = 0
 STATUS_FAILURE = 1
 STATUS_KEYBOARD_INTERRUPT = 130
 DEFAULT_BATCH_SIZE = 1000
+DEFAULT_MAX_PAGES = 10000
 
 
 def run_analysis_job(
@@ -150,6 +170,9 @@ def merge_module_sync_metadata(
     :param synced_type: The sub-module's type
     :param update_tag: Timestamp used to determine data freshness
     """
+    # Import here to avoid circular import with cartography.client.core.tx
+    from cartography.client.core.tx import run_write_query
+
     template = Template(
         """
         MERGE (n:ModuleSyncMetadata{id:'${group_type}_${group_id}_${synced_type}'})
@@ -161,7 +184,8 @@ def merge_module_sync_metadata(
             n.lastupdated=$UPDATE_TAG
     """,
     )
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         template.safe_substitute(
             group_type=group_type,
             group_id=group_id,
@@ -208,26 +232,28 @@ def aws_paginate(
     client: boto3.client,
     method_name: str,
     object_name: str,
+    max_pages: int | None = DEFAULT_MAX_PAGES,
     **kwargs: Any,
-) -> List[Dict]:
+) -> Iterable[Dict]:
     """
     Helper method for boilerplate boto3 pagination
     The **kwargs will be forwarded to the paginator
     """
     paginator = client.get_paginator(method_name)
-    items = []
-    i = 0
     for i, page in enumerate(paginator.paginate(**kwargs), start=1):
         if i % 100 == 0:
             logger.info(f"fetching page number {i}")
         if object_name in page:
-            items.extend(page[object_name])
+            items = page[object_name]
+            yield from items
         else:
             logger.warning(
                 f"""aws_paginate: Key "{object_name}" is not present, check if this is a typo.
 If not, then the AWS datatype somehow does not have this key.""",
             )
-    return items
+        if max_pages is not None and i >= max_pages:
+            logger.warning(f"Reached max batch size of {max_pages} pages")
+            break
 
 
 AWSGetFunc = TypeVar("AWSGetFunc", bound=Callable[..., Iterable])
@@ -241,15 +267,46 @@ AWSGetFunc = TypeVar("AWSGetFunc", bound=Callable[..., Iterable])
 
 def backoff_handler(details: Dict) -> None:
     """
-    Handler that will be executed on exception by backoff mechanism
+    Handler that will be executed on exception by backoff mechanism.
+
+    The backoff library may provide partial details (e.g. ``wait`` can be ``None`` when a
+    retry is triggered immediately). Format the message defensively so logging never raises.
     """
+    wait = details.get("wait")
+    if isinstance(wait, (int, float)):
+        wait_display = f"{wait:0.1f}"
+    elif wait is None:
+        wait_display = "unknown"
+    else:
+        wait_display = str(wait)
+
+    tries = details.get("tries")
+    tries_display = str(tries) if tries is not None else "unknown"
+
+    target = details.get("target", "<unknown>")
+
     logger.warning(
-        "Backing off {wait:0.1f} seconds after {tries} tries. Calling function {target}".format(
-            **details,
-        ),
+        "Backing off %s seconds after %s tries. Calling function %s",
+        wait_display,
+        tries_display,
+        target,
     )
 
 
+# Error codes that indicate a service is unavailable in a region or blocked by policies
+AWS_REGION_ACCESS_DENIED_ERROR_CODES = [
+    "AccessDenied",
+    "AccessDeniedException",
+    "AuthFailure",
+    "AuthorizationError",
+    "AuthorizationErrorException",
+    "InvalidClientTokenId",
+    "UnauthorizedOperation",
+    "UnrecognizedClientException",
+    "InternalServerErrorException",
+]
+
+
 # TODO Move this to cartography.intel.aws.util.common
 def aws_handle_regions(func: AWSGetFunc) -> AWSGetFunc:
     """
@@ -261,15 +318,6 @@ def aws_handle_regions(func: AWSGetFunc) -> AWSGetFunc:
 
     This should be used on `get_` functions that normally return a list of items.
     """
-    ERROR_CODES = [
-        "AccessDenied",
-        "AccessDeniedException",
-        "AuthFailure",
-        "InvalidClientTokenId",
-        "UnauthorizedOperation",
-        "UnrecognizedClientException",
-        "InternalServerErrorException",
-    ]
 
     @wraps(func)
     # fix for AWS TooManyRequestsException
@@ -279,7 +327,7 @@ def aws_handle_regions(func: AWSGetFunc) -> AWSGetFunc:
     # https://github.com/cartography-cncf/cartography/issues/25
     @backoff.on_exception(
         backoff.expo,
-        botocore.exceptions.ClientError,
+        (botocore.exceptions.ClientError, ResponseParserError),
         max_time=600,
         on_backoff=backoff_handler,
     )
@@ -287,17 +335,38 @@ def aws_handle_regions(func: AWSGetFunc) -> AWSGetFunc:
         try:
             return func(*args, **kwargs)
         except botocore.exceptions.ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code")
+            if error_code == "InvalidToken":
+                raise RuntimeError(
+                    "AWS returned an InvalidToken error. Configure regional STS endpoints by "
+                    "setting environment variable AWS_STS_REGIONAL_ENDPOINTS=regional or adding "
+                    "'sts_regional_endpoints = regional' to your AWS config file."
+                ) from e
             # The account is not authorized to use this service in this region
             # so we can continue without raising an exception
-            if e.response["Error"]["Code"] in ERROR_CODES:
-                logger.warning(
-                    "{} in this region. Skipping...".format(
-                        e.response["Error"]["Message"],
-                    ),
-                )
+            if error_code in AWS_REGION_ACCESS_DENIED_ERROR_CODES:
+                error_message = e.response.get("Error", {}).get("Message")
+                if is_service_control_policy_explicit_deny(e):
+                    logger.warning(
+                        "Service control policy denied access while calling %s: %s",
+                        func.__name__,
+                        error_message,
+                    )
+                else:
+                    logger.warning(
+                        "{} in this region. Skipping...".format(
+                            error_message,
+                        ),
+                    )
                 return []
             else:
                 raise
+        except EndpointConnectionError:
+            logger.warning(
+                "Encountered an EndpointConnectionError. This means that the AWS "
+                "resource is not available in this region. Skipping.",
+            )
+            return []
 
     return cast(AWSGetFunc, inner_function)
 
@@ -353,17 +422,18 @@ def camel_to_snake(name: str) -> str:
     return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
 
 
-def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> List[List]:
+def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> Iterable[List[Any]]:
     """
-    Takes an Iterable of items and returns a list of lists of the same items,
+    Takes an Iterable of items and returns a Generator of lists of the same items,
      batched into chunks of the provided `size`.
 
     Use:
     x = [1,2,3,4,5,6,7,8]
-    batch(x, size=3) -> [[1, 2, 3], [4, 5, 6], [7, 8]]
+    batch(x, size=3) -> Iterator yielding [1, 2, 3], [4, 5, 6], [7, 8]
     """
-    items = list(items)
-    return [items[i : i + size] for i in range(0, len(items), size)]
+    it = iter(items)
+    while chunk := list(islice(it, size)):
+        yield chunk
 
 
 def is_throttling_exception(exc: Exception) -> bool:

{cartography-0.104.0rc2.dist-info → cartography-0.123.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.104.0rc2
+Version: 0.123.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
-License: apache2
+License-Expression: Apache-2.0
 Project-URL: Homepage, https://cartography-cncf.github.io/cartography
 Project-URL: Documentation, https://cartography-cncf.github.io/cartography
 Project-URL: Repository, https://github.com/cartography-cncf/cartography
@@ -11,7 +11,6 @@ Project-URL: Issues, https://github.com/cartography-cncf/cartography/issues
 Project-URL: Changelog, https://github.com/cartography-cncf/cartography/releases
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
@@ -24,9 +23,11 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: backoff>=2.1.2
 Requires-Dist: boto3>=1.15.1
+Requires-Dist: aioboto3>=13.0.0
+Requires-Dist: httpx>=0.24.0
 Requires-Dist: botocore>=1.18.1
 Requires-Dist: dnspython>=1.15.0
-Requires-Dist: neo4j>=4.4.4
+Requires-Dist: neo4j<6.0.0,>=5.28.2
 Requires-Dist: policyuniverse>=1.1.0.0
 Requires-Dist: google-api-python-client>=1.7.8
 Requires-Dist: google-auth>=2.37.0
@@ -41,11 +42,21 @@ Requires-Dist: python-digitalocean>=1.16.0
 Requires-Dist: adal>=1.2.4
 Requires-Dist: azure-cli-core>=2.26.0
 Requires-Dist: azure-mgmt-compute>=5.0.0
+Requires-Dist: azure-mgmt-containerinstance>=10.0.0
 Requires-Dist: azure-mgmt-resource>=10.2.0
 Requires-Dist: azure-mgmt-cosmosdb>=6.0.0
+Requires-Dist: azure-mgmt-web>=7.0.0
+Requires-Dist: azure-mgmt-eventgrid>=10.0.0
+Requires-Dist: azure-mgmt-logic>=10.0.0
+Requires-Dist: azure-mgmt-datafactory>=8.0.0
+Requires-Dist: azure-mgmt-containerservice>=30.0.0
+Requires-Dist: azure-mgmt-network>=25.0.0
+Requires-Dist: azure-mgmt-security>=5.0.0
+Requires-Dist: azure-mgmt-monitor>=3.0.0
 Requires-Dist: msrestazure>=0.6.4
 Requires-Dist: azure-mgmt-storage>=16.0.0
 Requires-Dist: azure-mgmt-sql<=1.0.0
+Requires-Dist: azure-mgmt-authorization>=0.60.0
 Requires-Dist: azure-identity>=1.5.0
 Requires-Dist: msgraph-sdk
 Requires-Dist: kubernetes>=22.6.0
@@ -55,6 +66,12 @@ Requires-Dist: python-dateutil
 Requires-Dist: xmltodict
 Requires-Dist: duo-client
 Requires-Dist: cloudflare<5.0.0,>=4.1.0
+Requires-Dist: scaleway>=2.10.0
+Requires-Dist: google-cloud-resource-manager>=1.14.2
+Requires-Dist: google-cloud-asset>=1.0.0
+Requires-Dist: types-aiobotocore-ecr
+Requires-Dist: typer>=0.9.0
+Requires-Dist: slack-sdk>=3.37.0
 Dynamic: license-file
 
 ![Cartography](docs/root/images/logo-horizontal.png)
@@ -80,28 +97,35 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 
 ## Supported platforms
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - API Gateway, CloudWatch, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
-- [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
-- [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
-- [Oracle Cloud Infrastructure](https://cartography-cncf.github.io/cartography/modules/oci/index.html) - IAM
-- [Okta](https://cartography-cncf.github.io/cartography/modules/okta/index.html) - users, groups, organizations, roles, applications, factors, trusted origins, reply URIs
-- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
-- [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
-- [Microsoft Azure](https://cartography-cncf.github.io/cartography/modules/azure/index.html) -  CosmosDB, SQL, Storage, Virtual Machine
-- [Microsoft Entra ID](https://cartography-cncf.github.io/cartography/modules/entra/index.html) -  Users
-- [Kubernetes](https://cartography-cncf.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container
-- [PagerDuty](https://cartography-cncf.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
-- [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs
-- [NIST CVE](https://cartography-cncf.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database
-- [Lastpass](https://cartography-cncf.github.io/cartography/modules/lastpass/index.html) - users
+- [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including multi-arch images, image layers, and attestations), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
+- [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
 - [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
+- [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord
+- [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs
+- [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
 - [Duo](https://cartography-cncf.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
+- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams, dependency graph manifests, dependencies
+- [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Bigtable, Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
+- [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups (deprecated - use Google Workspace instead)
+- [Google Workspace](https://cartography-cncf.github.io/cartography/modules/googleworkspace/index.html) - users, groups, devices, OAuth apps
 - [Kandji](https://cartography-cncf.github.io/cartography/modules/kandji/index.html) - Devices
+- [Keycloak](https://cartography-cncf.github.io/cartography/modules/keycloak/index.html) - Realms, Users, Groups, Roles, Scopes, Clients, IdentityProviders, Authentication Flows, Authentication Executions, Organizations, Organization Domains
+- [Kubernetes](https://cartography-cncf.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container, ServiceAccount, Role, RoleBinding, ClusterRole, ClusterRoleBinding, OIDCProvider
+- [Lastpass](https://cartography-cncf.github.io/cartography/modules/lastpass/index.html) - users
+- [Microsoft Azure](https://cartography-cncf.github.io/cartography/modules/azure/index.html) - App Service, Container Instance, CosmosDB, Data Factory, Event Grid, Functions, Azure Kubernetes Service (AKS), Load Balancer, Logic Apps, Resource Group, SQL, Storage, Virtual Machine, Virtual Networks
+- [Microsoft Entra ID](https://cartography-cncf.github.io/cartography/modules/entra/index.html) -  Users, Groups, Applications, OUs, App Roles, federation to AWS Identity Center
+- [NIST CVE](https://cartography-cncf.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database
+- [Okta](https://cartography-cncf.github.io/cartography/modules/okta/index.html) - users, groups, organizations, roles, applications, factors, trusted origins, reply URIs, federation to AWS roles, federation to AWS Identity Center
+- [OpenAI](https://cartography-cncf.github.io/cartography/modules/openai/index.html) - Organization, AdminApiKey, User, Project, ServiceAccount, ApiKey
+- [Oracle Cloud Infrastructure](https://cartography-cncf.github.io/cartography/modules/oci/index.html) - IAM
+- [PagerDuty](https://cartography-cncf.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
+- [Scaleway](https://cartography-cncf.github.io/cartography/modules/scaleway/index.html) - Projects, IAM, Local Storage, Instances
+- [SentinelOne](https://cartography-cncf.github.io/cartography/modules/sentinelone/index.html) - Accounts, Agents, Applications, Application Versions, CVEs
+- [Slack](https://cartography-cncf.github.io/cartography/modules/slack/index.html) - Teams, Users, UserGroups, Channels
 - [SnipeIT](https://cartography-cncf.github.io/cartography/modules/snipeit/index.html) - Users, Assets
 - [Tailscale](https://cartography-cncf.github.io/cartography/modules/tailscale/index.html) - Tailnet, Users, Devices, Groups, Tags, PostureIntegrations
-- [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord
-- [OpenAI](https://cartography-cncf.github.io/cartography/modules/openai/index.html) - Organization, AdminApiKey, User, Project, ServiceAccount, ApiKey
-- [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
+- [Trivy Scanner](https://cartography-cncf.github.io/cartography/modules/trivy/index.html) - AWS ECR Images
 
 
 ## Philosophy
@@ -137,6 +161,17 @@ When you are ready to try it in production, read [here](https://cartography-cncf
 
 ## Usage
 
+### Running rules
+
+You can check your environment against common security frameworks using the `cartography-rules` command.
+
+```bash
+cartography-rules run all
+```
+
+See [the rules docs](https://cartography-cncf.github.io/cartography/usage/rules.html) for more detail.
+
+
 ### Querying the database directly
 
 ![poweruser.png](docs/root/images/poweruser.png)
@@ -147,6 +182,10 @@ Now that data is in the graph, you can quickly start with our [querying tutorial
 Directly querying Neo4j is already very useful as a sort of "swiss army knife" for security data problems, but you can also build applications and data pipelines around Cartography. View this doc on [applications](https://cartography-cncf.github.io/cartography/usage/applications.html).
 
 
+## Docs
+
+See [here](https://cartography-cncf.github.io/cartography/)
+
 ## Community
 
 - Hang out with us on Slack: Join the CNCF Slack workspace [here](https://communityinviter.com/apps/cloud-native/cncf), and then join the `#cartography` channel.
@@ -162,7 +201,7 @@ This project is licensed under the [Apache 2.0 License](LICENSE).
 Thank you for considering contributing to Cartography!
 
 ### Code of conduct
-All contributors and participants of this project must follow the  [CNCF code of conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
+All contributors and participants of this project must follow the [CNCF code of conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
 
 ### Bug reports and feature requests and discussions
 Submit a GitHub issue to report a bug or request a new feature. If we decide that the issue needs more discussion - usually because the scope is too large or we need to make careful decision - we will convert the issue to a [GitHub Discussion](https://github.com/cartography-cncf/cartography/discussions).