cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.104.0rc2'
-__version_tuple__ = version_tuple = (0, 104, 0, 'rc2')
+__version__ = version = '0.123.0'
+__version_tuple__ = version_tuple = (0, 123, 0)
+
+__commit_id__ = commit_id = None

cartography/cli.py

@@ -71,8 +71,8 @@ class CLI:
             default="bolt://localhost:7687",
             help=(
                 "A valid Neo4j URI to sync against. See "
-                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
-                "structure of a Neo4j URI."
+                "https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme for complete "
+                "documentation on the structure of a Neo4j URI."
             ),
         )
         parser.add_argument(
@@ -182,6 +182,14 @@ class CLI:
                 "syncing other accounts and delay raising an exception until the very end."
             ),
         )
+        parser.add_argument(
+            "--aws-cloudtrail-management-events-lookback-hours",
+            type=int,
+            default=None,
+            help=(
+                "Number of hours back to retrieve CloudTrail management events from. If not specified, CloudTrail management events will not be retrieved."
+            ),
+        )
         parser.add_argument(
             "--oci-sync-all-profiles",
             action="store_true",
@@ -226,6 +234,11 @@ class CLI:
                 "The name of environment variable containing Azure Client Secret for Service Principal Authentication."
             ),
         )
+        parser.add_argument(
+            "--azure-subscription-id",
+            type=str,
+            help="The Azure Subscription ID to sync.",
+        )
         parser.add_argument(
             "--entra-tenant-id",
             type=str,
@@ -256,6 +269,27 @@ class CLI:
                 " If not specified, cartography by default will run all AWS sync modules available."
             ),
         )
+        parser.add_argument(
+            "--aws-guardduty-severity-threshold",
+            type=str,
+            default=None,
+            help=(
+                "GuardDuty severity threshold filter. Only findings at or above this severity level will be synced. "
+                "Valid values: LOW, MEDIUM, HIGH, CRITICAL. If not specified, all findings (except archived) will be synced. "
+                "Example: 'HIGH' will sync only HIGH and CRITICAL findings, filtering out LOW and MEDIUM severity findings."
+            ),
+        )
+        parser.add_argument(
+            "--experimental-aws-inspector-batch",
+            type=int,
+            default=1000,
+            help=(
+                "EXPERIMENTAL: This feature is experimental and may be removed in the future. "
+                "Batch size for AWS Inspector findings sync. Controls how many findings are fetched, processed and cleaned up at a time. "
+                "Default is 1000. Increase this value if you have a large number of findings and want to reduce API calls, "
+                "or decrease it if you're experiencing memory issues."
+            ),
+        )
         parser.add_argument(
             "--analysis-job-directory",
             type=str,
@@ -305,6 +339,14 @@ class CLI:
                 "Required if you are using the GitHub intel module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--github-commit-lookback-days",
+            type=int,
+            default=30,
+            help=(
+                "Number of days to look back for tracking GitHub users committing to repositories. Defaults to 30 days."
+            ),
+        )
         parser.add_argument(
             "--digitalocean-token-env-var",
             type=str,
@@ -323,6 +365,24 @@ class CLI:
                 "If omitted the default permission relationships will be created"
             ),
         )
+        parser.add_argument(
+            "--azure-permission-relationships-file",
+            type=str,
+            default="cartography/data/azure_permission_relationships.yaml",
+            help=(
+                "The path to the Azure permission relationships mapping file."
+                "If omitted the default Azure permission relationships will be created"
+            ),
+        )
+        parser.add_argument(
+            "--gcp-permission-relationships-file",
+            type=str,
+            default="cartography/data/gcp_permission_relationships.yaml",
+            help=(
+                "The path to the GCP permission relationships mapping file. "
+                "If omitted the default GCP permission relationships will be used"
+            ),
+        )
         parser.add_argument(
             "--jamf-base-uri",
             type=str,
@@ -376,6 +436,12 @@ class CLI:
                 "The path to kubeconfig file specifying context to access K8s cluster(s)."
             ),
         )
+        parser.add_argument(
+            "--managed-kubernetes",
+            default=None,
+            type=str,
+            help=("Type of managed Kubernetes service (e.g., 'eks'). Optional."),
+        )
         parser.add_argument(
             "--nist-cve-url",
             type=str,
@@ -482,6 +548,24 @@ class CLI:
                 "The name of environment variable containing secrets for GSuite authentication."
             ),
         )
+        parser.add_argument(
+            "--googleworkspace-auth-method",
+            type=str,
+            default="delegated",
+            choices=["delegated", "oauth", "default"],
+            help=(
+                'Google Workspace authentication method. Can be "delegated" for service account or "oauth" for OAuth. '
+                '"Default" best if using gcloud CLI.'
+            ),
+        )
+        parser.add_argument(
+            "--googleworkspace-tokens-env-var",
+            type=str,
+            default="GOOGLEWORKSPACE_GOOGLE_APPLICATION_CREDENTIALS",
+            help=(
+                "The name of environment variable containing secrets for Google Workspace authentication."
+            ),
+        )
         parser.add_argument(
             "--lastpass-cid-env-var",
             type=str,
@@ -637,6 +721,257 @@ class CLI:
                 "Required if you are using the Anthropic intel module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--airbyte-client-id",
+            type=str,
+            default=None,
+            help=(
+                "The Airbyte client ID to use for authentication. "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--airbyte-client-secret-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing the Airbyte client secret for authentication. "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--airbyte-api-url",
+            type=str,
+            default="https://api.airbyte.com/v1",
+            help=(
+                "The base URL for the Airbyte API (default is the public Airbyte Cloud API). "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--trivy-s3-bucket",
+            type=str,
+            default=None,
+            help=(
+                "The S3 bucket name containing Trivy scan results. "
+                "Required if you are using the Trivy module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--trivy-s3-prefix",
+            type=str,
+            default=None,
+            help=(
+                "The S3 prefix path containing Trivy scan results. "
+                "Required if you are using the Trivy module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--ontology-users-source",
+            type=str,
+            default=None,
+            help=(
+                "Comma-separated list of sources of truth for user data in the ontology. "
+                "'User' nodes will only be created for users that exist in one of the sources. "
+                "Required if you are using the ontology module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--ontology-devices-source",
+            type=str,
+            default=None,
+            help=(
+                "Comma-separated list of sources of truth for client computer data in the ontology. "
+                "'Device' nodes will only be created for groups that exist in one of the sources. "
+                "Required if you are using the ontology module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--trivy-results-dir",
+            type=str,
+            default=None,
+            help=(
+                "Path to a directory containing Trivy JSON results on disk. "
+                "Required if you are using the Trivy module with local results."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-org",
+            type=str,
+            default=None,
+            help=(
+                "The Scaleway organization ID to sync. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-access-key",
+            type=str,
+            default=None,
+            help=(
+                "The Scaleway access key to use for authentication. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-secret-key-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing the Scaleway secret key for authentication. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--sentinelone-account-ids",
+            type=str,
+            default=None,
+            help=(
+                "Comma-separated list of SentinelOne account IDs to sync. "
+                "If not specified, all accessible accounts will be synced."
+            ),
+        )
+        parser.add_argument(
+            "--sentinelone-api-url",
+            type=str,
+            default=None,
+            help=(
+                "SentinelOne API URL. Required if you are using the SentinelOne intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--sentinelone-api-token-env-var",
+            type=str,
+            default="SENTINELONE_API_TOKEN",
+            help=(
+                "The name of an environment variable containing the SentinelOne API token. "
+                "Required if you are using the SentinelOne intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-client-id",
+            type=str,
+            default=None,
+            help=(
+                "The Keycloak client ID to sync. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-client-secret-env-var",
+            type=str,
+            default="KEYCLOAK_CLIENT_SECRET",
+            help=(
+                "The name of an environment variable containing the Keycloak client secret. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-url",
+            type=str,
+            help=(
+                "The base URL for the Keycloak instance. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise. "
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-realm",
+            type=str,
+            default="master",
+            help=(
+                "The Keycloak realm used for authentication (note: all available realms will be synced). "
+                "Should be `master` (default value) in most of the cases. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise. "
+            ),
+        )
+        parser.add_argument(
+            "--slack-token-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of environment variable containing the Slack Token. "
+                "Required if you are using the Slack intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--slack-teams",
+            type=str,
+            default=None,
+            help=(
+                "The Slack Team ID to sync, comma separated. If not provided, all accessible teams will be synced. "
+            ),
+        )
+        parser.add_argument(
+            "--slack-channels-memberships",
+            action="store_true",
+            help=("Pull memberships for Slack Channels (can be time consuming)."),
+        )
+        parser.add_argument(
+            "--spacelift-api-endpoint",
+            type=str,
+            default=None,
+            help=(
+                "Spacelift GraphQL API endpoint (e.g., https://yourorg.app.spacelift.io/graphql). "
+                "Required if you are using the Spacelift intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--spacelift-api-token-env-var",
+            type=str,
+            default="SPACELIFT_API_TOKEN",
+            help=(
+                "The name of an environment variable containing the Spacelift API token. "
+                "Alternative to using API key ID/secret. Ignored if API key credentials are provided."
+            ),
+        )
+        parser.add_argument(
+            "--spacelift-api-key-id-env-var",
+            type=str,
+            default="SPACELIFT_API_KEY_ID",
+            help=(
+                "The name of an environment variable containing the Spacelift API key ID. "
+                "Use with --spacelift-api-key-secret-env-var for automatic token exchange. "
+                "Alternative to providing a pre-generated token."
+            ),
+        )
+        parser.add_argument(
+            "--spacelift-api-key-secret-env-var",
+            type=str,
+            default="SPACELIFT_API_KEY_SECRET",
+            help=(
+                "The name of an environment variable containing the Spacelift API key secret. "
+                "Use with --spacelift-api-key-id-env-var for automatic token exchange. "
+                "Alternative to providing a pre-generated token."
+            ),
+        )
+        parser.add_argument(
+            "--spacelift-ec2-ownership-aws-profile",
+            type=str,
+            default=None,
+            help=(
+                "AWS profile name to use for fetching EC2 ownership data from S3. "
+                "Optional. If not provided, uses default AWS credentials. "
+            ),
+        )
+        parser.add_argument(
+            "--spacelift-ec2-ownership-s3-bucket",
+            type=str,
+            default=None,
+            help=(
+                "S3 bucket name containing CloudTrail data for EC2 ownership relationships. "
+                "Required for EC2 ownership sync (along with --spacelift-ec2-ownership-s3-prefix)."
+            ),
+        )
+        parser.add_argument(
+            "--spacelift-ec2-ownership-s3-prefix",
+            type=str,
+            default=None,
+            help=(
+                "S3 prefix for CloudTrail data for EC2 ownership relationships. "
+                "All JSON files under this prefix will be processed. "
+                "Required for EC2 ownership sync (along with --spacelift-ec2-ownership-s3-bucket)."
+            ),
+        )
 
         return parser
 
@@ -787,8 +1122,8 @@ class CLI:
                 logger.warning("A Kandji base URI was provided but a token was not.")
                 config.kandji_token = None
         else:
-            logger.warning("A Kandji base URI was not provided.")
             config.kandji_base_uri = None
+            config.kandji_token = None
 
         if config.statsd_enabled:
             logger.debug(
@@ -835,6 +1170,17 @@ class CLI:
         else:
             config.gsuite_tokens_env_var = None
 
+        # Google Workspace config
+        if config.googleworkspace_tokens_env_var:
+            logger.debug(
+                f"Reading config string for Google Workspace from environment variable {config.googleworkspace_tokens_env_var}",
+            )
+            config.googleworkspace_config = os.environ.get(
+                config.googleworkspace_tokens_env_var
+            )
+        else:
+            config.googleworkspace_tokens_env_var = None
+
         # Lastpass config
         if config.lastpass_cid_env_var:
             logger.debug(
@@ -914,10 +1260,10 @@ class CLI:
                 config.snipeit_token = os.environ.get("SNIPEIT_TOKEN")
             else:
                 logger.warning("A SnipeIT base URI was provided but a token was not.")
-                config.kandji_token = None
+                config.snipeit_token = None
         else:
-            logger.warning("A SnipeIT base URI was not provided.")
             config.snipeit_base_uri = None
+            config.snipeit_token = None
 
         # Tailscale config
         if config.tailscale_token_env_var:
@@ -955,6 +1301,121 @@ class CLI:
         else:
             config.anthropic_apikey = None
 
+        # Airbyte config
+        if config.airbyte_client_id and config.airbyte_client_secret_env_var:
+            logger.debug(
+                f"Reading Airbyte client secret from environment variable {config.airbyte_client_secret_env_var}",
+            )
+            config.airbyte_client_secret = os.environ.get(
+                config.airbyte_client_secret_env_var,
+            )
+        else:
+            config.airbyte_client_secret = None
+
+        # Trivy config
+        if config.trivy_s3_bucket:
+            logger.debug(f"Trivy S3 bucket: {config.trivy_s3_bucket}")
+
+        if config.trivy_s3_prefix:
+            logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
+
+        if config.trivy_results_dir:
+            logger.debug(f"Trivy results dir: {config.trivy_results_dir}")
+
+        # Scaleway config
+        if config.scaleway_secret_key_env_var:
+            logger.debug(
+                f"Reading Scaleway secret key from environment variable {config.scaleway_secret_key_env_var}",
+            )
+            config.scaleway_secret_key = os.environ.get(
+                config.scaleway_secret_key_env_var,
+            )
+        else:
+            config.scaleway_secret_key = None
+
+        # SentinelOne config
+        if config.sentinelone_account_ids:
+            config.sentinelone_account_ids = [
+                id.strip() for id in config.sentinelone_account_ids.split(",")
+            ]
+            logger.debug(
+                f"Parsed {len(config.sentinelone_account_ids)} SentinelOne account IDs to sync"
+            )
+        else:
+            config.sentinelone_account_ids = None
+
+        if config.sentinelone_api_url and config.sentinelone_api_token_env_var:
+            logger.debug(
+                f"Reading API token for SentinelOne from environment variable {config.sentinelone_api_token_env_var}",
+            )
+            config.sentinelone_api_token = os.environ.get(
+                config.sentinelone_api_token_env_var
+            )
+        else:
+            config.sentinelone_api_token = None
+
+        # Keycloak config
+        if config.keycloak_client_secret_env_var:
+            logger.debug(
+                f"Reading Client Secret for Keycloak from environment variable {config.keycloak_client_secret_env_var}",
+            )
+            config.keycloak_client_secret = os.environ.get(
+                config.keycloak_client_secret_env_var
+            )
+        else:
+            config.keycloak_client_secret = None
+
+        # Slack config
+        if config.slack_token_env_var:
+            logger.debug(
+                f"Reading Slack token from environment variable {config.slack_token_env_var}",
+            )
+            config.slack_token = os.environ.get(config.slack_token_env_var)
+        else:
+            config.slack_token = None
+
+        # Spacelift config
+        # Read endpoint from CLI arg or env var
+        if not config.spacelift_api_endpoint:
+            config.spacelift_api_endpoint = os.environ.get("SPACELIFT_API_ENDPOINT")
+
+        if config.spacelift_api_endpoint:
+            # Try to read API token
+            if config.spacelift_api_token_env_var:
+                logger.debug(
+                    f"Reading API token for Spacelift from environment variable {config.spacelift_api_token_env_var}",
+                )
+                config.spacelift_api_token = os.environ.get(
+                    config.spacelift_api_token_env_var
+                )
+            else:
+                config.spacelift_api_token = None
+
+            # Try to read API key ID and secret
+            if config.spacelift_api_key_id_env_var:
+                logger.debug(
+                    f"Reading API key ID for Spacelift from environment variable {config.spacelift_api_key_id_env_var}",
+                )
+                config.spacelift_api_key_id = os.environ.get(
+                    config.spacelift_api_key_id_env_var
+                )
+            else:
+                config.spacelift_api_key_id = None
+
+            if config.spacelift_api_key_secret_env_var:
+                logger.debug(
+                    f"Reading API key secret for Spacelift from environment variable {config.spacelift_api_key_secret_env_var}",
+                )
+                config.spacelift_api_key_secret = os.environ.get(
+                    config.spacelift_api_key_secret_env_var
+                )
+            else:
+                config.spacelift_api_key_secret = None
+        else:
+            config.spacelift_api_token = None
+            config.spacelift_api_key_id = None
+            config.spacelift_api_key_secret = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/client/aws/__init__.py

@@ -0,0 +1,19 @@
+from typing import List
+
+import neo4j
+
+from cartography.client.core.tx import read_list_of_values_tx
+from cartography.util import timeit
+
+
+@timeit
+def list_accounts(neo4j_session: neo4j.Session) -> List[str]:
+    """
+    :param neo4j_session: The neo4j session object.
+    :return: A list of all AWS account IDs in the graph
+    """
+    # See https://community.neo4j.com/t/extract-list-of-nodes-and-labels-from-path/13665/4
+    query = """
+    MATCH (a:AWSAccount) RETURN a.id
+    """
+    return neo4j_session.read_transaction(read_list_of_values_tx, query)

cartography/client/aws/ecr.py

@@ -0,0 +1,51 @@
+from typing import Set
+from typing import Tuple
+
+import neo4j
+
+from cartography.client.core.tx import read_list_of_tuples_tx
+from cartography.util import timeit
+
+
+@timeit
+def get_ecr_images(
+    neo4j_session: neo4j.Session, aws_account_id: str
+) -> Set[Tuple[str, str, str, str, str]]:
+    """
+    Queries the graph for all ECR images and their parent images.
+    Returns 5-tuples of ECR repository regions, tags, URIs, names, and binary digests. This is used to identify which
+    images to scan.
+    :param neo4j_session: The neo4j session object.
+    :param aws_account_id: The AWS account ID to get ECR repo data for.
+    :return: 5-tuples of repo region, image tag, image URI, repo_name, and image_digest.
+    """
+    # See https://community.neo4j.com/t/extract-list-of-nodes-and-labels-from-path/13665/4
+    query = """
+MATCH (e1:ECRRepositoryImage)<-[:REPO_IMAGE]-(repo:ECRRepository)
+MATCH (repo)<-[:RESOURCE]-(:AWSAccount {id: $AWS_ID})
+
+// OPTIONAL traversal of parent hierarchy
+OPTIONAL MATCH path = (e1)-[:PARENT*1..]->(ancestor:ECRRepositoryImage)
+WITH e1,
+     CASE
+         WHEN path IS NULL THEN [e1]
+         ELSE [n IN nodes(path) | n] + [e1]
+     END AS repo_img_collection_unflattened
+
+// Flatten and dedupe
+UNWIND repo_img_collection_unflattened AS repo_img
+WITH DISTINCT repo_img
+
+// Match image metadata
+MATCH (er:ECRRepository)-[:REPO_IMAGE]->(repo_img)-[:IMAGE]->(img:ECRImage)
+
+RETURN DISTINCT
+    er.region AS region,
+    repo_img.tag AS tag,
+    repo_img.id AS uri,
+    er.name AS repo_name,
+    img.digest AS digest
+    """
+    return neo4j_session.read_transaction(
+        read_list_of_tuples_tx, query, AWS_ID=aws_account_id
+    )