cartography 0.94.0rc3__tar.gz → 0.95.0__tar.gz

{cartography-0.94.0rc3 → cartography-0.95.0}/LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2019 Lyft, Inc.
+   Copyright 2024 The Linux Foundation
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{cartography-0.94.0rc3/cartography.egg-info → cartography-0.95.0}/PKG-INFO

@@ -1,10 +1,9 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.94.0rc3
+Version: 0.95.0
 Summary: Explore assets and their relationships across your technical infrastructure.
-Home-page: https://www.github.com/lyft/cartography
-Maintainer: Lyft
-Maintainer-email: security@lyft.com
+Home-page: https://www.github.com/cartography-cncf/cartography
+Maintainer: Cartography Contributors
 License: apache2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -18,7 +17,6 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Description-Content-Type: text/markdown
 License-File: LICENSE
-License-File: NOTICE
 Requires-Dist: backoff>=2.1.2
 Requires-Dist: boto3>=1.15.1
 Requires-Dist: botocore>=1.18.1

cartography-0.95.0/README.md

@@ -0,0 +1,122 @@
+![Cartography](docs/root/images/logo-horizontal.png)
+
+Cartography is a Python tool that consolidates infrastructure assets and the relationships between them in an intuitive graph view powered by a [Neo4j](https://www.neo4j.com) database.
+
+![Visualization of RDS nodes and AWS nodes](docs/root/images/accountsandrds.png)
+
+## Why Cartography?
+Cartography aims to enable a broad set of exploration and automation scenarios. It is particularly good at exposing otherwise hidden dependency relationships between your service's assets so that you may validate assumptions about security risks.
+
+Service owners can generate asset reports, Red Teamers can discover attack paths, and Blue Teamers can identify areas for security improvement. All can benefit from using the graph for manual exploration through a web frontend interface, or in an automated fashion by calling the APIs.
+
+Cartography is not the only [security](https://github.com/dowjones/hammer) [graph](https://github.com/BloodHoundAD/BloodHound) [tool](https://github.com/Netflix/security_monkey) [out](https://github.com/vysecurity/ANGRYPUPPY) [there](https://github.com/duo-labs/cloudmapper), but it differentiates itself by being fully-featured yet generic and [extensible](https://cartography-cncf.github.io/cartography/dev/writing-analysis-jobs.html) enough to help make anyone better understand their risk exposure, regardless of what platforms they use. Rather than being focused on one core scenario or attack vector like the other linked tools, Cartography focuses on flexibility and exploration.
+
+You can learn more about the story behind Cartography in our [presentation at BSidesSF 2019](https://www.youtube.com/watch?v=ZukUmZSKSek).
+
+
+## Supported platforms
+
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - API Gateway, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager, Security Hub, SQS, SSM, STS, Tags
+- [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
+- [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
+- [Oracle Cloud Infrastructure](docs/setup/config/oci.md) - IAM
+- [Okta](https://cartography-cncf.github.io/cartography/modules/okta/index.html) - users, groups, organizations, roles, applications, factors, trusted origins, reply URIs
+- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
+- [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
+- [Microsoft Azure](https://cartography-cncf.github.io/cartography/modules/azure/index.html) -  CosmosDB, SQL, Storage, Virtual Machine
+- [Kubernetes](https://cartography-cncf.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container
+- [PagerDuty](https://cartography-cncf.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
+- [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs
+- [NIST CVE](https://cartography-cncf.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database
+- [Lastpass](https://cartography-cncf.github.io/cartography/modules/lastpass/index.html) - users
+- [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
+- [Duo](https://cartography-cncf.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
+- [Kandji](https://cartography-cncf.github.io/cartography/modules/kandji/index.html) - Devices
+- [SnipeIT](https://cartography-cncf.github.io/cartography/modules/snipeit/index.html) - Users, Assets
+
+
+## Philosophy
+Here are some points that can help you decide if adopting Cartography is a good fit for your problem.
+
+### What Cartography is
+- A simple Python script that pulls data from multiple providers and writes it to a Neo4j graph database in batches.
+- A powerful analysis tool that captures the current snapshot of the environment, building a uniquely useful inventory where you can ask complex questions such as:
+  - Which identities have access to which datastores?
+  - What are the cross-tenant permission relationships in the environment?
+  - What are the network paths in and out of the environment?
+  - What are the backup policies for my datastores?
+- Battle-tested in production by [many companies](#who-uses-cartography).
+- Straightforward to extend with your own custom plugins.
+- Provides a useful data-plane that you can build automation and CSPM (Cloud Security Posture Management) applications on top of.
+
+### What Cartography is not
+- A near-real time capability.
+  - Cartography is not designed for very fast updates. Cartography writes to the database in a batches (not streamed).
+  - Cartography is also limited by how most upstream sources only provide APIs to retrieve assets in a batched manner.
+- By itself, Cartography does not capture data changes over time.
+  - Although we do include a [drift detection](https://cartography-cncf.github.io/cartography/usage/drift-detect.html) feature.
+  - It's also possible to implement other processes in your Cartography installation to make this happen.
+
+
+## Install and configure
+
+### Trying out Cartography on a test machine
+Start [here](https://cartography-cncf.github.io/cartography/install.html) to set up a test graph and get data into it.
+
+### Setting up Cartography in production
+When you are ready to try it in production, read [here](https://cartography-cncf.github.io/cartography/ops.html) for recommendations on getting cartography spun up in your environment.
+
+## Usage
+
+### Querying the database directly
+
+![poweruser.png](docs/root/images/poweruser.png)
+
+Now that data is in the graph, you can quickly start with our [querying tutorial](https://cartography-cncf.github.io/cartography/usage/tutorial.html). Our [data schema](https://cartography-cncf.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
+
+### Building applications around Cartography
+Directly querying Neo4j is already very useful as a sort of "swiss army knife" for security data problems, but you can also build applications and data pipelines around Cartography. View this doc on [applications](https://cartography-cncf.github.io/cartography/usage/applications.html).
+
+
+## Community
+
+- Join us on `#cartography` on the [Lyft OSS Slack](https://join.slack.com/t/lyftoss/shared_invite/enQtOTYzODg5OTQwNDE2LTFiYjgwZWM3NTNhMTFkZjc4Y2IxOTI4NTdiNTdhNjQ4M2Q5NTIzMjVjOWI4NmVlNjRiZmU2YzA5NTc3MmFjYTQ).
+- Talk to us and see what we're working on at our [monthly community meeting](https://calendar.google.com/calendar/embed?src=lyft.com_p10o6ceuiieq9sqcn1ef61v1io%40group.calendar.google.com&ctz=America%2FLos_Angeles).
+  - Meeting minutes are [here](https://docs.google.com/document/d/1VyRKmB0dpX185I15BmNJZpfAJ_Ooobwz0U1WIhjDxvw).
+  - Recorded videos are posted [here](https://www.youtube.com/playlist?list=PLMga2YJvAGzidUWJB_fnG7EHI4wsDDsE1).
+- Our current project roadmap is [here](https://github.com/orgs/lyft/projects/26/views/1).
+
+## License
+
+This project is licensed under the [Apache 2.0 License](LICENSE).
+
+## Contributing
+Thank you for considering contributing to Cartography!
+
+### Code of conduct
+All contributors and participants of this project must follow the  [CNCF code of conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
+
+### Bug reports and feature requests and discussions
+Submit a GitHub issue to report a bug or request a new feature. If we decide that the issue needs more discussion - usually because the scope is too large or we need to make careful decision - we will convert the issue to a [GitHub Discussion](https://github.com/lyft/cartography/discussions).
+
+### Developing Cartography
+
+Get started with our [developer documentation](https://cartography-cncf.github.io/cartography/dev/developer-guide.html). Please feel free to submit your own PRs to update documentation if you've found a better way to explain something.
+
+## Who uses Cartography?
+
+1. [Lyft](https://www.lyft.com)
+1. [Thought Machine](https://thoughtmachine.net/)
+1. [MessageBird](https://messagebird.com)
+1. [Cloudanix](https://www.cloudanix.com/)
+1. [Corelight](https://www.corelight.com/)
+1. {Your company here} :-)
+
+If your organization uses Cartography, please file a PR and update this list. Say hi on Slack too!
+
+---
+
+Cartography is a [Cloud Native Computing Foundation](https://www.cncf.io/) sandbox project.<br>
+<div style="background-color: white; display: inline-block; padding: 10px;">
+  <img src="docs/root/images/cncf-color.png" alt="CNCF Logo" width="200">
+</div>

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/cli.py

@@ -220,23 +220,6 @@ class CLI:
                 ' If not specified, cartography by default will run all AWS sync modules available.'
             ),
         )
-        parser.add_argument(
-            '--crxcavator-api-base-uri',
-            type=str,
-            default='https://api.crxcavator.io/v1',
-            help=(
-                'Base URI for the CRXcavator API. Defaults to public API endpoint.'
-            ),
-        )
-        parser.add_argument(
-            '--crxcavator-api-key-env-var',
-            type=str,
-            default=None,
-            help=(
-                'The name of an environment variable containing a key with which to auth to the CRXcavator API. '
-                'Required if you are using the CRXcavator intel module. Ignored otherwise.'
-            ),
-        )
         parser.add_argument(
             '--analysis-job-directory',
             type=str,
@@ -541,6 +524,28 @@ class CLI:
                 'Required if you are using the Semgrep intel module. Ignored otherwise.'
             ),
         )
+        parser.add_argument(
+            '--snipeit-base-uri',
+            type=str,
+            default=None,
+            help=(
+                'Your SnipeIT base URI'
+                'Required if you are using the SnipeIT intel module. Ignored otherwise.'
+            ),
+        )
+        parser.add_argument(
+            '--snipeit-token-env-var',
+            type=str,
+            default=None,
+            help='The name of an environment variable containing token with which to authenticate to SnipeIT.',
+        )
+        parser.add_argument(
+            '--snipeit-tenant-id',
+            type=str,
+            default=None,
+            help='An ID for the SnipeIT tenant.',
+        )
+
         return parser
 
     def main(self, argv: str) -> int:
@@ -604,13 +609,6 @@ class CLI:
         else:
             config.okta_api_key = None
 
-        # CRXcavator config
-        if config.crxcavator_api_base_uri and config.crxcavator_api_key_env_var:
-            logger.debug(f"Reading API key for CRXcavator from env variable {config.crxcavator_api_key_env_var}.")
-            config.crxcavator_api_key = os.environ.get(config.crxcavator_api_key_env_var)
-        else:
-            config.crxcavator_api_key = None
-
         # GitHub config
         if config.github_config_env_var:
             logger.debug(f"Reading config string for GitHub from environment variable {config.github_config_env_var}")
@@ -744,6 +742,26 @@ class CLI:
         else:
             config.cve_api_key = None
 
+        # SnipeIT config
+        if config.snipeit_base_uri:
+            if config.snipeit_token_env_var:
+                logger.debug(
+                    "Reading SnipeIT API token from environment variable '%s'.",
+                    config.snipeit_token_env_var,
+                )
+                config.snipeit_token = os.environ.get(config.snipeit_token_env_var)
+            elif os.environ.get('SNIPEIT_TOKEN'):
+                logger.debug(
+                    "Reading SnipeIT API token from environment variable 'SNIPEIT_TOKEN'.",
+                )
+                config.snipeit_token = os.environ.get('SNIPEIT_TOKEN')
+            else:
+                logger.warning("A SnipeIT base URI was provided but a token was not.")
+                config.kandji_token = None
+        else:
+            logger.warning("A SnipeIT base URI was not provided.")
+            config.snipeit_base_uri = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/config.py

@@ -43,10 +43,6 @@ class Config:
     :param azure_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
-    :type crxcavator_api_base_uri: str
-    :param crxcavator_api_base_uri: URI for CRXcavator API. Optional.
-    :type crxcavator_api_key: str
-    :param crxcavator_api_key: Auth key for CRXcavator API. Optional.
     :type analysis_job_directory: str
     :param analysis_job_directory: Path to a directory tree containing analysis jobs to run. Optional.
     :type oci_sync_all_profiles: bool
@@ -111,6 +107,12 @@ class Config:
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     :param semgrep_app_token: The Semgrep api token. Optional.
     :type semgrep_app_token: str
+    :type snipeit_base_uri: string
+    :param snipeit_base_uri: SnipeIT data provider base URI. Optional.
+    :type snipeit_token: string
+    :param snipeit_token: Token used to authenticate to the SnipeIT data provider. Optional.
+    :type snipeit_tenant_id: string
+    :param snipeit_tenant_id: Token used to authenticate to the SnipeIT data provider. Optional.
     """
 
     def __init__(
@@ -131,8 +133,6 @@ class Config:
         azure_client_secret=None,
         aws_requested_syncs=None,
         analysis_job_directory=None,
-        crxcavator_api_base_uri=None,
-        crxcavator_api_key=None,
         oci_sync_all_profiles=None,
         okta_org_id=None,
         okta_api_key=None,
@@ -170,6 +170,9 @@ class Config:
         duo_api_secret=None,
         duo_api_hostname=None,
         semgrep_app_token=None,
+        snipeit_base_uri=None,
+        snipeit_token=None,
+        snipeit_tenant_id=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -187,8 +190,6 @@ class Config:
         self.azure_client_secret = azure_client_secret
         self.aws_requested_syncs = aws_requested_syncs
         self.analysis_job_directory = analysis_job_directory
-        self.crxcavator_api_base_uri = crxcavator_api_base_uri
-        self.crxcavator_api_key = crxcavator_api_key
         self.oci_sync_all_profiles = oci_sync_all_profiles
         self.okta_org_id = okta_org_id
         self.okta_api_key = okta_api_key
@@ -226,3 +227,6 @@ class Config:
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
         self.semgrep_app_token = semgrep_app_token
+        self.snipeit_base_uri = snipeit_base_uri
+        self.snipeit_token = snipeit_token
+        self.snipeit_tenant_id = snipeit_tenant_id

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/data/indexes.cypher

@@ -65,8 +65,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.accesskeyid);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AutoScalingGroup) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:AutoScalingGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ChromeExtension) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ChromeExtension) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:CrowdstrikeHost) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:CrowdstrikeHost) ON (n.instance_id);
 CREATE INDEX IF NOT EXISTS FOR (n:CrowdstrikeHost) ON (n.lastupdated);

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/driftdetect/cli.py

@@ -30,7 +30,7 @@ class CLI:
                 'graph database and reports the deviations.'
             ),
             epilog='For more documentation please visit: '
-                   'https://github.com/lyft/cartography/blob/master/docs/drift-detect.md',
+                   'https://cartography-cncf.github.io/cartography/usage/drift-detect.html',
         )
         parser.add_argument(
             '-v',

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/graph/job.py

@@ -150,7 +150,14 @@ class GraphJob:
             )
 
         statements: List[GraphStatement] = [
-            GraphStatement(query, parameters=parameters, iterative=True, iterationsize=100) for query in queries
+            GraphStatement(
+                query,
+                parameters=parameters,
+                iterative=True,
+                iterationsize=100,
+                parent_job_name=node_schema.label,
+                parent_job_sequence_num=idx,
+            ) for idx, query in enumerate(queries, start=1)
         ]
 
         return cls(

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/intel/aws/permission_relationships.py

@@ -322,8 +322,12 @@ def cleanup_rpr(
     )
 
     statement = GraphStatement(
-        cleanup_rpr_query_template, {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_id},
-        True, 1000,
+        cleanup_rpr_query_template,
+        {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_id},
+        True,
+        1000,
+        parent_job_name=f"{relationship_name}:{node_label}",
+        parent_job_sequence_num=1,
     )
     statement.run(neo4j_session)
 

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/intel/gcp/__init__.py

@@ -120,11 +120,11 @@ def _initialize_resources(credentials: GoogleCredentials) -> Resource:
     return Resources(
         crm_v1=_get_crm_resource_v1(credentials),
         crm_v2=_get_crm_resource_v2(credentials),
-        compute=_get_compute_resource(credentials),
-        storage=_get_storage_resource(credentials),
-        container=_get_container_resource(credentials),
         serviceusage=_get_serviceusage_resource(credentials),
-        dns=_get_dns_resource(credentials),
+        compute=None,
+        container=None,
+        dns=None,
+        storage=None,
     )
 
 
@@ -159,12 +159,12 @@ def _services_enabled_on_project(serviceusage: Resource, project_id: str) -> Set
         return set()
 
 
-def _sync_single_project(
+def _sync_single_project_compute(
     neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
-    Handles graph sync for a single GCP project.
+    Handles graph sync for a single GCP project on Compute resources.
     :param neo4j_session: The Neo4j session
     :param resources: namedtuple of the GCP resource objects
     :param project_id: The project ID number to sync.  See  the `projectId` field in
@@ -175,14 +175,72 @@ def _sync_single_project(
     """
     # Determine the resources available on the project.
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
+    compute_cred = _get_compute_resource(get_gcp_credentials())
     if service_names.compute in enabled_services:
-        compute.sync(neo4j_session, resources.compute, project_id, gcp_update_tag, common_job_parameters)
+        compute.sync(neo4j_session, compute_cred, project_id, gcp_update_tag, common_job_parameters)
+
+
+def _sync_single_project_storage(
+    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Handles graph sync for a single GCP project on Storage resources.
+    :param neo4j_session: The Neo4j session
+    :param resources: namedtuple of the GCP resource objects
+    :param project_id: The project ID number to sync.  See  the `projectId` field in
+    https://cloud.google.com/resource-manager/reference/rest/v1/projects
+    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Other parameters sent to Neo4j
+    :return: Nothing
+    """
+    # Determine the resources available on the project.
+    enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
+    storage_cred = _get_storage_resource(get_gcp_credentials())
     if service_names.storage in enabled_services:
-        storage.sync_gcp_buckets(neo4j_session, resources.storage, project_id, gcp_update_tag, common_job_parameters)
+        storage.sync_gcp_buckets(neo4j_session, storage_cred, project_id, gcp_update_tag, common_job_parameters)
+
+
+def _sync_single_project_gke(
+    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Handles graph sync for a single GCP project GKE resources.
+    :param neo4j_session: The Neo4j session
+    :param resources: namedtuple of the GCP resource objects
+    :param project_id: The project ID number to sync.  See  the `projectId` field in
+    https://cloud.google.com/resource-manager/reference/rest/v1/projects
+    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Other parameters sent to Neo4j
+    :return: Nothing
+    """
+    # Determine the resources available on the project.
+    enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
+    container_cred = _get_container_resource(get_gcp_credentials())
     if service_names.gke in enabled_services:
-        gke.sync_gke_clusters(neo4j_session, resources.container, project_id, gcp_update_tag, common_job_parameters)
+        gke.sync_gke_clusters(neo4j_session, container_cred, project_id, gcp_update_tag, common_job_parameters)
+
+
+def _sync_single_project_dns(
+    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Handles graph sync for a single GCP project DNS resources.
+    :param neo4j_session: The Neo4j session
+    :param resources: namedtuple of the GCP resource objects
+    :param project_id: The project ID number to sync.  See  the `projectId` field in
+    https://cloud.google.com/resource-manager/reference/rest/v1/projects
+    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Other parameters sent to Neo4j
+    :return: Nothing
+    """
+    # Determine the resources available on the project.
+    enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
+    dns_cred = _get_dns_resource(get_gcp_credentials())
     if service_names.dns in enabled_services:
-        dns.sync(neo4j_session, resources.dns, project_id, gcp_update_tag, common_job_parameters)
+        dns.sync(neo4j_session, dns_cred, project_id, gcp_update_tag, common_job_parameters)
 
 
 def _sync_multiple_projects(
@@ -203,26 +261,38 @@ def _sync_multiple_projects(
     """
     logger.info("Syncing %d GCP projects.", len(projects))
     crm.sync_gcp_projects(neo4j_session, projects, gcp_update_tag, common_job_parameters)
+    # Compute data sync
+    for project in projects:
+        project_id = project['projectId']
+        logger.info("Syncing GCP project %s for Compute.", project_id)
+        _sync_single_project_compute(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
 
+    # Storage data sync
     for project in projects:
         project_id = project['projectId']
-        logger.info("Syncing GCP project %s.", project_id)
-        _sync_single_project(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        logger.info("Syncing GCP project %s for Storage", project_id)
+        _sync_single_project_storage(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+
+    # GKE data sync
+    for project in projects:
+        project_id = project['projectId']
+        logger.info("Syncing GCP project %s for GKE", project_id)
+        _sync_single_project_gke(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+
+    # DNS data sync
+    for project in projects:
+        project_id = project['projectId']
+        logger.info("Syncing GCP project %s for DNS", project_id)
+        _sync_single_project_dns(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
 
 
 @timeit
-def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+def get_gcp_credentials() -> GoogleCredentials:
     """
-    Starts the GCP ingestion process by initializing Google Application Default Credentials, creating the necessary
-    resource objects, listing all GCP organizations and projects available to the GCP identity, and supplying that
-    context to all intel modules.
-    :param neo4j_session: The Neo4j session
-    :param config: A `cartography.config` object
-    :return: Nothing
+    Gets access tokens for GCP API access.
+    :param: None
+    :return: GoogleCredentials
     """
-    common_job_parameters = {
-        "UPDATE_TAG": config.update_tag,
-    }
     try:
         # Explicitly use Application Default Credentials.
         # See https://oauth2client.readthedocs.io/en/latest/source/
@@ -239,7 +309,24 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
             ),
             e,
         )
-        return
+        return credentials
+
+
+@timeit
+def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    Starts the GCP ingestion process by initializing Google Application Default Credentials, creating the necessary
+    resource objects, listing all GCP organizations and projects available to the GCP identity, and supplying that
+    context to all intel modules.
+    :param neo4j_session: The Neo4j session
+    :param config: A `cartography.config` object
+    :return: Nothing
+    """
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    credentials = get_gcp_credentials()
 
     resources = _initialize_resources(credentials)
 

{cartography-0.94.0rc3 → cartography-0.95.0}/cartography/intel/kandji/__init__.py

@@ -21,7 +21,7 @@ def start_kandji_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
     """
     if config.kandji_base_uri is None or config.kandji_token is None or config.kandji_tenant_id is None:
         logger.warning(
-            'Required parameter(s) missing. Skipping sync.',
+            'Required parameter missing. Skipping sync. '
             'See docs to configure.',
         )
         return

cartography-0.95.0/cartography/intel/semgrep/__init__.py

@@ -0,0 +1,30 @@
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.semgrep.dependencies import sync_dependencies
+from cartography.intel.semgrep.deployment import sync_deployment
+from cartography.intel.semgrep.findings import sync_findings
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_semgrep_ingestion(
+    neo4j_session: neo4j.Session, config: Config,
+) -> None:
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+    if not config.semgrep_app_token:
+        logger.info('Semgrep import is not configured - skipping this module. See docs to configure.')
+        return
+
+    # sync_deployment must be called first since it populates common_job_parameters
+    # with the deployment ID and slug, which are required by the other sync functions
+    sync_deployment(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+    sync_dependencies(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+    sync_findings(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)