cartography 0.104.0rc3__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.104.0rc3'
-__version_tuple__ = version_tuple = (0, 104, 0, 'rc3')
+__version__ = version = '0.106.0'
+__version_tuple__ = version_tuple = (0, 106, 0)

cartography/cli.py

@@ -71,8 +71,8 @@ class CLI:
             default="bolt://localhost:7687",
             help=(
                 "A valid Neo4j URI to sync against. See "
-                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
-                "structure of a Neo4j URI."
+                "https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme for complete "
+                "documentation on the structure of a Neo4j URI."
             ),
         )
         parser.add_argument(
@@ -637,6 +637,78 @@ class CLI:
                 "Required if you are using the Anthropic intel module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--airbyte-client-id",
+            type=str,
+            default=None,
+            help=(
+                "The Airbyte client ID to use for authentication. "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--airbyte-client-secret-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing the Airbyte client secret for authentication. "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--airbyte-api-url",
+            type=str,
+            default="https://api.airbyte.com/v1",
+            help=(
+                "The base URL for the Airbyte API (default is the public Airbyte Cloud API). "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--trivy-s3-bucket",
+            type=str,
+            default=None,
+            help=(
+                "The S3 bucket name containing Trivy scan results. "
+                "Required if you are using the Trivy module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--trivy-s3-prefix",
+            type=str,
+            default=None,
+            help=(
+                "The S3 prefix path containing Trivy scan results. "
+                "Required if you are using the Trivy module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-org",
+            type=str,
+            default=None,
+            help=(
+                "The Scaleway organization ID to sync. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-access-key",
+            type=str,
+            default=None,
+            help=(
+                "The Scaleway access key to use for authentication. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-secret-key-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing the Scaleway secret key for authentication. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
 
         return parser
 
@@ -914,7 +986,7 @@ class CLI:
                 config.snipeit_token = os.environ.get("SNIPEIT_TOKEN")
             else:
                 logger.warning("A SnipeIT base URI was provided but a token was not.")
-                config.kandji_token = None
+                config.snipeit_token = None
         else:
             logger.warning("A SnipeIT base URI was not provided.")
             config.snipeit_base_uri = None
@@ -955,6 +1027,35 @@ class CLI:
         else:
             config.anthropic_apikey = None
 
+        # Airbyte config
+        if config.airbyte_client_id and config.airbyte_client_secret_env_var:
+            logger.debug(
+                f"Reading Airbyte client secret from environment variable {config.airbyte_client_secret_env_var}",
+            )
+            config.airbyte_client_secret = os.environ.get(
+                config.airbyte_client_secret_env_var,
+            )
+        else:
+            config.airbyte_client_secret = None
+
+        # Trivy config
+        if config.trivy_s3_bucket:
+            logger.debug(f"Trivy S3 bucket: {config.trivy_s3_bucket}")
+
+        if config.trivy_s3_prefix:
+            logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
+
+        # Scaleway config
+        if config.scaleway_secret_key_env_var:
+            logger.debug(
+                f"Reading Scaleway secret key from environment variable {config.scaleway_secret_key_env_var}",
+            )
+            config.scaleway_secret_key = os.environ.get(
+                config.scaleway_secret_key_env_var,
+            )
+        else:
+            config.scaleway_secret_key = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/client/aws/__init__.py

@@ -0,0 +1,19 @@
+from typing import List
+
+import neo4j
+
+from cartography.client.core.tx import read_list_of_values_tx
+from cartography.util import timeit
+
+
+@timeit
+def list_accounts(neo4j_session: neo4j.Session) -> List[str]:
+    """
+    :param neo4j_session: The neo4j session object.
+    :return: A list of all AWS account IDs in the graph
+    """
+    # See https://community.neo4j.com/t/extract-list-of-nodes-and-labels-from-path/13665/4
+    query = """
+    MATCH (a:AWSAccount) RETURN a.id
+    """
+    return neo4j_session.read_transaction(read_list_of_values_tx, query)

cartography/client/aws/ecr.py

@@ -0,0 +1,51 @@
+from typing import Set
+from typing import Tuple
+
+import neo4j
+
+from cartography.client.core.tx import read_list_of_tuples_tx
+from cartography.util import timeit
+
+
+@timeit
+def get_ecr_images(
+    neo4j_session: neo4j.Session, aws_account_id: str
+) -> Set[Tuple[str, str, str, str, str]]:
+    """
+    Queries the graph for all ECR images and their parent images.
+    Returns 5-tuples of ECR repository regions, tags, URIs, names, and binary digests. This is used to identify which
+    images to scan.
+    :param neo4j_session: The neo4j session object.
+    :param aws_account_id: The AWS account ID to get ECR repo data for.
+    :return: 5-tuples of repo region, image tag, image URI, repo_name, and image_digest.
+    """
+    # See https://community.neo4j.com/t/extract-list-of-nodes-and-labels-from-path/13665/4
+    query = """
+MATCH (e1:ECRRepositoryImage)<-[:REPO_IMAGE]-(repo:ECRRepository)
+MATCH (repo)<-[:RESOURCE]-(:AWSAccount {id: $AWS_ID})
+
+// OPTIONAL traversal of parent hierarchy
+OPTIONAL MATCH path = (e1)-[:PARENT*1..]->(ancestor:ECRRepositoryImage)
+WITH e1,
+     CASE
+         WHEN path IS NULL THEN [e1]
+         ELSE [n IN nodes(path) | n] + [e1]
+     END AS repo_img_collection_unflattened
+
+// Flatten and dedupe
+UNWIND repo_img_collection_unflattened AS repo_img
+WITH DISTINCT repo_img
+
+// Match image metadata
+MATCH (er:ECRRepository)-[:REPO_IMAGE]->(repo_img)-[:IMAGE]->(img:ECRImage)
+
+RETURN DISTINCT
+    er.region AS region,
+    repo_img.tag AS tag,
+    repo_img.id AS uri,
+    er.name AS repo_name,
+    img.digest AS digest
+    """
+    return neo4j_session.read_transaction(
+        read_list_of_tuples_tx, query, AWS_ID=aws_account_id
+    )

cartography/client/core/tx.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any
 from typing import Dict
 from typing import List
@@ -8,10 +9,15 @@ from typing import Union
 import neo4j
 
 from cartography.graph.querybuilder import build_create_index_queries
+from cartography.graph.querybuilder import build_create_index_queries_for_matchlink
 from cartography.graph.querybuilder import build_ingestion_query
+from cartography.graph.querybuilder import build_matchlink_query
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelSchema
 from cartography.util import batch
 
+logger = logging.getLogger(__name__)
+
 
 def read_list_of_values_tx(
     tx: neo4j.Transaction,
@@ -255,6 +261,25 @@ def ensure_indexes(
         neo4j_session.run(query)
 
 
+def ensure_indexes_for_matchlinks(
+    neo4j_session: neo4j.Session,
+    rel_schema: CartographyRelSchema,
+) -> None:
+    """
+    Creates indexes for node fields if they don't exist for the given CartographyRelSchema object.
+    This is only used for load_rels() where we match on and connect existing nodes.
+    This is not used for CartographyNodeSchema objects.
+    """
+    queries = build_create_index_queries_for_matchlink(rel_schema)
+    logger.debug(f"CREATE INDEX queries for {rel_schema.rel_label}: {queries}")
+    for query in queries:
+        if not query.startswith("CREATE INDEX IF NOT EXISTS"):
+            raise ValueError(
+                'Query provided to `ensure_indexes_for_matchlinks()` does not start with "CREATE INDEX IF NOT EXISTS".',
+            )
+        neo4j_session.run(query)
+
+
 def load(
     neo4j_session: neo4j.Session,
     node_schema: CartographyNodeSchema,
@@ -276,3 +301,40 @@ def load(
     ensure_indexes(neo4j_session, node_schema)
     ingestion_query = build_ingestion_query(node_schema)
     load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)
+
+
+def load_matchlinks(
+    neo4j_session: neo4j.Session,
+    rel_schema: CartographyRelSchema,
+    dict_list: list[dict[str, Any]],
+    **kwargs,
+) -> None:
+    """
+    Main entrypoint for intel modules to write relationships to the graph between two existing nodes.
+    :param neo4j_session: The Neo4j session
+    :param rel_schema: The CartographyRelSchema object to generate a query.
+    :param dict_list: The data to load to the graph represented as a list of dicts. The dicts must contain the source and
+    target node ids.
+    :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
+    :return: None
+    """
+    if len(dict_list) == 0:
+        # If there is no data to load, save some time.
+        return
+
+    # Validate that required kwargs are provided for cleanup queries
+    if "_sub_resource_label" not in kwargs:
+        raise ValueError(
+            f"Required kwarg '_sub_resource_label' not provided for {rel_schema.rel_label}. "
+            "This is needed for cleanup queries."
+        )
+    if "_sub_resource_id" not in kwargs:
+        raise ValueError(
+            f"Required kwarg '_sub_resource_id' not provided for {rel_schema.rel_label}. "
+            "This is needed for cleanup queries."
+        )
+
+    ensure_indexes_for_matchlinks(neo4j_session, rel_schema)
+    matchlink_query = build_matchlink_query(rel_schema)
+    logger.debug(f"Matchlink query: {matchlink_query}")
+    load_graph_data(neo4j_session, matchlink_query, dict_list, **kwargs)

cartography/config.py

@@ -137,6 +137,22 @@ class Config:
     :param openai_org_id: OpenAI organization id. Optional.
     :type anthropic_apikey: string
     :param anthropic_apikey: Anthropic API key. Optional.
+    :type airbyte_client_id: str
+    :param airbyte_client_id: Airbyte client ID for API authentication. Optional.
+    :type airbyte_client_secret: str
+    :param airbyte_client_secret: Airbyte client secret for API authentication. Optional.
+    :type airbyte_api_url: str
+    :param airbyte_api_url: Airbyte API base URL, e.g. https://api.airbyte.com/v1. Optional.
+    :type trivy_s3_bucket: str
+    :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
+    :type trivy_s3_prefix: str
+    :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
+    :type scaleway_access_key: str
+    :param scaleway_access_key: Scaleway access key. Optional.
+    :type scaleway_secret_key: str
+    :param scaleway_secret_key: Scaleway secret key. Optional.
+    :type scaleway_org: str
+    :param scaleway_org: Scaleway organization id. Optional.
     """
 
     def __init__(
@@ -209,6 +225,14 @@ class Config:
         openai_apikey=None,
         openai_org_id=None,
         anthropic_apikey=None,
+        airbyte_client_id=None,
+        airbyte_client_secret=None,
+        airbyte_api_url=None,
+        trivy_s3_bucket=None,
+        trivy_s3_prefix=None,
+        scaleway_access_key=None,
+        scaleway_secret_key=None,
+        scaleway_org=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -278,3 +302,11 @@ class Config:
         self.openai_apikey = openai_apikey
         self.openai_org_id = openai_org_id
         self.anthropic_apikey = anthropic_apikey
+        self.airbyte_client_id = airbyte_client_id
+        self.airbyte_client_secret = airbyte_client_secret
+        self.airbyte_api_url = airbyte_api_url
+        self.trivy_s3_bucket = trivy_s3_bucket
+        self.trivy_s3_prefix = trivy_s3_prefix
+        self.scaleway_access_key = scaleway_access_key
+        self.scaleway_secret_key = scaleway_secret_key
+        self.scaleway_org = scaleway_org

cartography/data/indexes.cypher

@@ -99,21 +99,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ECRRepositoryImage) ON (n.tag);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRRepositoryImage) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRScanFinding) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRScanFinding) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSCluster) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerInstance) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerInstance) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSService) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSService) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTask) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTask) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.lastupdated);
@@ -227,9 +212,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:OCITenancy) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:OCIUser) ON (n.ocid);
 CREATE INDEX IF NOT EXISTS FOR (n:OCIUser) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:OCIUser) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:Package) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:Package) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:Package) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:PagerDutyEscalationPolicy) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:PagerDutyEscalationPolicy) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:PagerDutyEscalationPolicy) ON (n.lastupdated);
@@ -378,22 +360,3 @@ CREATE INDEX IF NOT EXISTS FOR (n:AzureDisk) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureDisk) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureSnapshot) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureSnapshot) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesCluster) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesCluster) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesNamespace) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesNamespace) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesNamespace) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesPod) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesPod) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesPod) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.image);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.lastupdated);

cartography/data/jobs/cleanup/aws_import_lambda_cleanup.json

@@ -31,7 +31,7 @@
       "iterationsize": 100
     },
     {
-      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSLambda)-[r:STS_ASSUME_ROLE_ALLOW]->(:AWSPrincipal) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r",
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSLambda)-[r:STS_ASSUMEROLE_ALLOW]->(:AWSPrincipal) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r",
       "iterative": true,
       "iterationsize": 100
     },

cartography/driftdetect/cli.py

@@ -63,8 +63,9 @@ class CLI:
             default="bolt://localhost:7687",
             help=(
                 "A valid Neo4j URI to sync against. See "
-                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
-                "structure of a Neo4j URI."
+                "https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme for "
+                "documentation on the structure of a Neo4j URI, and "
+                "https://neo4j.com/docs/api/python-driver/current/ for complete documentation on the Python driver."
             ),
         )
         parser_get_state.add_argument(