cartography 0.93.0__tar.gz → 0.94.0__tar.gz

{cartography-0.93.0/cartography.egg-info → cartography-0.94.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.93.0
+Version: 0.94.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.93.0 → cartography-0.94.0}/README.md

@@ -34,7 +34,8 @@ Start [here](https://lyft.github.io/cartography/install.html).
 - [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users
 - [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
-
+- [Kandji](https://lyft.github.io/cartography/modules/kandji/index.html) - Devices
+- [SnipeIT](https://lyft.github.io/cartography/modules/snipeit/index.html) - Users, Assets
 
 ## Usage
 Start with our [tutorial](https://lyft.github.io/cartography/usage/tutorial.html). Our [data schema](https://lyft.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
@@ -74,6 +75,7 @@ and follow the instructions to sign the CLA.
 1. [MessageBird](https://messagebird.com)
 1. [Cloudanix](https://www.cloudanix.com/)
 1. [ZeusCloud](https://www.zeuscloud.io/)
+1. [Corelight](https://www.corelight.com/)
 1. {Your company here} :-)
 
 If your organization uses Cartography, please file a PR and update this list. Say hi on Slack too!

{cartography-0.93.0 → cartography-0.94.0}/cartography/cli.py

@@ -541,6 +541,28 @@ class CLI:
                 'Required if you are using the Semgrep intel module. Ignored otherwise.'
             ),
         )
+        parser.add_argument(
+            '--snipeit-base-uri',
+            type=str,
+            default=None,
+            help=(
+                'Your SnipeIT base URI'
+                'Required if you are using the SnipeIT intel module. Ignored otherwise.'
+            ),
+        )
+        parser.add_argument(
+            '--snipeit-token-env-var',
+            type=str,
+            default=None,
+            help='The name of an environment variable containing token with which to authenticate to SnipeIT.',
+        )
+        parser.add_argument(
+            '--snipeit-tenant-id',
+            type=str,
+            default=None,
+            help='An ID for the SnipeIT tenant.',
+        )
+
         return parser
 
     def main(self, argv: str) -> int:
@@ -744,6 +766,26 @@ class CLI:
         else:
             config.cve_api_key = None
 
+        # SnipeIT config
+        if config.snipeit_base_uri:
+            if config.snipeit_token_env_var:
+                logger.debug(
+                    "Reading SnipeIT API token from environment variable '%s'.",
+                    config.snipeit_token_env_var,
+                )
+                config.snipeit_token = os.environ.get(config.snipeit_token_env_var)
+            elif os.environ.get('SNIPEIT_TOKEN'):
+                logger.debug(
+                    "Reading SnipeIT API token from environment variable 'SNIPEIT_TOKEN'.",
+                )
+                config.snipeit_token = os.environ.get('SNIPEIT_TOKEN')
+            else:
+                logger.warning("A SnipeIT base URI was provided but a token was not.")
+                config.kandji_token = None
+        else:
+            logger.warning("A SnipeIT base URI was not provided.")
+            config.snipeit_base_uri = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

{cartography-0.93.0 → cartography-0.94.0}/cartography/config.py

@@ -111,6 +111,12 @@ class Config:
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     :param semgrep_app_token: The Semgrep api token. Optional.
     :type semgrep_app_token: str
+    :type snipeit_base_uri: string
+    :param snipeit_base_uri: SnipeIT data provider base URI. Optional.
+    :type snipeit_token: string
+    :param snipeit_token: Token used to authenticate to the SnipeIT data provider. Optional.
+    :type snipeit_tenant_id: string
+    :param snipeit_tenant_id: Token used to authenticate to the SnipeIT data provider. Optional.
     """
 
     def __init__(
@@ -170,6 +176,9 @@ class Config:
         duo_api_secret=None,
         duo_api_hostname=None,
         semgrep_app_token=None,
+        snipeit_base_uri=None,
+        snipeit_token=None,
+        snipeit_tenant_id=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -226,3 +235,6 @@ class Config:
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
         self.semgrep_app_token = semgrep_app_token
+        self.snipeit_base_uri = snipeit_base_uri
+        self.snipeit_token = snipeit_token
+        self.snipeit_tenant_id = snipeit_tenant_id

{cartography-0.93.0 → cartography-0.94.0}/cartography/data/jobs/scoped_analysis/semgrep_sca_risk_analysis.json

@@ -13,47 +13,47 @@
         },
         {
             "__comment__": "not possible to identify if reachable && version specifier is the only flag of the vulnerability (likelihood = rare) && severity in [low, medium, high] -> Risk = Info",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'UNKNOWN_EXPOSURE', reachability_check:'VERSION_SPECIFIER', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity IN ['LOW', 'MEDIUM', 'HIGH'] SET s.reachability_risk = 'INFO' return COUNT(*) as TotalCompleted",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'UNREACHABLE', reachability_check:'NO REACHABILITY ANALYSIS', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity IN ['LOW', 'MEDIUM', 'HIGH'] SET s.reachability_risk = 'INFO' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
             "__comment__": "not possible to identify if reachable && version specifier is the only flag of the vulnerability (likelihood = rare) && severity = critical -> Risk = Low",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'UNKNOWN_EXPOSURE', reachability_check:'VERSION_SPECIFIER', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'CRITICAL' SET s.reachability_risk = 'LOW' return COUNT(*) as TotalCompleted",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'UNREACHABLE', reachability_check:'NO REACHABILITY ANALYSIS', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'CRITICAL' SET s.reachability_risk = 'LOW' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
-            "__comment__": "manual review required to confirm && version specifier is the only flag of the vulnerability (likelihood = possible) && severity in [low, medium] -> Risk = Low",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'MANUAL_REVIEW_REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity IN ['LOW', 'MEDIUM'] SET s.reachability_risk = 'LOW' return COUNT(*) as TotalCompleted",
+            "__comment__": "manual review required to confirm exploitation when conditions met && identified version is vulnerable (likelihood = possible) && severity in [low, medium] -> Risk = Low",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'CONDITIONALLY REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity IN ['LOW', 'MEDIUM'] SET s.reachability_risk = 'LOW' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
-            "__comment__": "manual review required to confirm && version specifier is the only flag of the vulnerability (likelihood = possible) && severity = high -> Risk = Medium",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'MANUAL_REVIEW_REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'HIGH' SET s.reachability_risk = 'MEDIUM' return COUNT(*) as TotalCompleted",
+            "__comment__": "manual review required to confirm exploitation when conditions met && identified version is vulnerable (likelihood = possible) && severity = high -> Risk = Medium",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'CONDITIONALLY REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'HIGH' SET s.reachability_risk = 'MEDIUM' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
-            "__comment__": "manual review required to confirm && version specifier is the only flag of the vulnerability (likelihood = possible) &&  severity = critical -> Risk = High",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'MANUAL_REVIEW_REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'CRITICAL' SET s.reachability_risk = 'HIGH' return COUNT(*) as TotalCompleted",
+            "__comment__": "manual review required to confirm exploitation when conditions met && identified version is vulnerable (likelihood = possible) &&  severity = critical -> Risk = High",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'CONDITIONALLY REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'CRITICAL' SET s.reachability_risk = 'HIGH' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
             "__comment__": "adding the vulnerable version flags it reachable (likelihood = likely) && severity in [low, medium] -> Risk = Low",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'ALWAYS_REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity IN ['LOW','MEDIUM'] SET s.reachability_risk = 'LOW' return COUNT(*) as TotalCompleted",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'ALWAYS REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity IN ['LOW','MEDIUM'] SET s.reachability_risk = 'LOW' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
-            "__comment__": "adding the vulnerable version flags it reachable (likelihood = likely) && severity = high -> Risk = Low",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'ALWAYS_REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'HIGH' SET s.reachability_risk = 'MEDIUM' return COUNT(*) as TotalCompleted",
+            "__comment__": "adding the vulnerable version flags it reachable (likelihood = likely) && severity = high -> Risk = Medium",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'ALWAYS REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'HIGH' SET s.reachability_risk = 'MEDIUM' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
             "__comment__": "adding the vulnerable version flags it reachable (special case for critical, if something is so critical that needs to be fixed, likelihood = likely)) && severity = critical -> Risk = Critical",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'ALWAYS_REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'CRITICAL' SET s.reachability_risk = 'CRITICAL' return COUNT(*) as TotalCompleted",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'ALWAYS REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) WHERE s.severity = 'CRITICAL' SET s.reachability_risk = 'CRITICAL' return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
             "__comment__": "if reachability analysis confirmed that is rechable (likelihood = certain) -> Risk = Severity",
-            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'REACHABILITY', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) SET s.reachability_risk = s.severity return COUNT(*) as TotalCompleted",
+            "query": "MATCH (g:GitHubRepository{archived:false})<-[:FOUND_IN]-(s:SemgrepSCAFinding{reachability:'REACHABLE', reachability_check:'REACHABLE', lastupdated:$UPDATE_TAG})<-[:RESOURCE]-(:SemgrepDeployment{id:$DEPLOYMENT_ID}) SET s.reachability_risk = s.severity return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {

{cartography-0.93.0 → cartography-0.94.0}/cartography/graph/job.py

@@ -150,7 +150,14 @@ class GraphJob:
             )
 
         statements: List[GraphStatement] = [
-            GraphStatement(query, parameters=parameters, iterative=True, iterationsize=100) for query in queries
+            GraphStatement(
+                query,
+                parameters=parameters,
+                iterative=True,
+                iterationsize=100,
+                parent_job_name=node_schema.label,
+                parent_job_sequence_num=idx,
+            ) for idx, query in enumerate(queries, start=1)
         ]
 
         return cls(

{cartography-0.93.0 → cartography-0.94.0}/cartography/intel/aws/inspector.py

@@ -18,6 +18,15 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# As of 7/22/24, Inspector is only available in the below regions. We will need to update this hardcoded list here over
+# time. :\ https://docs.aws.amazon.com/general/latest/gr/inspector2.html
+AWS_INSPECTOR_REGIONS = {
+    "us-east-2", "us-east-1", "us-west-1", "us-west-2", "af-south-1", "ap-east-1", "ap-southeast-3", "ap-south-1",
+    "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1",
+    "eu-central-1", "eu-west-1", "eu-west-2", "eu-south-1", "eu-west-3", "eu-north-1", "eu-central-2", "me-south-1",
+    "sa-east-1",
+}
+
 
 @timeit
 @aws_handle_regions
@@ -206,7 +215,9 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    for region in regions:
+    inspector_regions = [region for region in regions if region in AWS_INSPECTOR_REGIONS]
+
+    for region in inspector_regions:
         logger.info(f"Syncing AWS Inspector findings for account {current_aws_account_id} and region {region}")
         findings = get_inspector_findings(boto3_session, region, current_aws_account_id)
         finding_data, package_data = transform_inspector_findings(findings)

{cartography-0.93.0 → cartography-0.94.0}/cartography/intel/aws/permission_relationships.py

@@ -322,8 +322,12 @@ def cleanup_rpr(
     )
 
     statement = GraphStatement(
-        cleanup_rpr_query_template, {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_id},
-        True, 1000,
+        cleanup_rpr_query_template,
+        {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_id},
+        True,
+        1000,
+        parent_job_name=f"{relationship_name}:{node_label}",
+        parent_job_sequence_num=1,
     )
     statement.run(neo4j_session)
 

{cartography-0.93.0 → cartography-0.94.0}/cartography/intel/github/teams.py

@@ -1,4 +1,6 @@
 import logging
+from collections import namedtuple
+from time import sleep
 from typing import Any
 from typing import Dict
 from typing import List
@@ -15,6 +17,8 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+RepoPermission = namedtuple('RepoPermission', ['repo_url', 'permission'])
+
 
 @timeit
 def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData, Dict[str, Any]]:
@@ -45,26 +49,53 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
 
 @timeit
 def _get_team_repos_for_multiple_teams(
-        team_raw_data: List[Dict[str, Any]],
+        team_raw_data: list[dict[str, Any]],
         org: str,
         api_url: str,
         token: str,
-) -> Dict[str, Any]:
-    result = {}
+) -> dict[str, list[RepoPermission]]:
+    result: dict[str, list[RepoPermission]] = {}
     for team in team_raw_data:
         team_name = team['slug']
         repo_count = team['repositories']['totalCount']
 
-        team_repos = _get_team_repos(org, api_url, token, team_name) if repo_count > 0 else None
+        if repo_count == 0:
+            # This team has access to no repos so let's move on
+            result[team_name] = []
+            continue
 
         repo_urls = []
         repo_permissions = []
-        if team_repos:
-            repo_urls = [t['url'] for t in team_repos.nodes] if team_repos.nodes else []
-            repo_permissions = [t['permission'] for t in team_repos.edges] if team_repos.edges else []
+
+        max_tries = 5
+
+        for current_try in range(1, max_tries + 1):
+            team_repos = _get_team_repos(org, api_url, token, team_name)
+
+            try:
+                # The `or []` is because `.nodes` can be None. See:
+                # https://docs.github.com/en/graphql/reference/objects#teamrepositoryconnection
+                for repo in team_repos.nodes or []:
+                    repo_urls.append(repo['url'])
+
+                # The `or []` is because `.edges` can be None.
+                for edge in team_repos.edges or []:
+                    repo_permissions.append(edge['permission'])
+                # We're done! Break out of the retry loop.
+                break
+
+            except TypeError:
+                # Handles issue #1334
+                logger.warning(
+                    f"GitHub returned None when trying to find repo or permission data for team {team_name}.",
+                    exc_info=True,
+                )
+                if current_try == max_tries:
+                    raise RuntimeError(f"GitHub returned a None repo url for team {team_name}, retries exhausted.")
+                sleep(current_try ** 2)
 
         # Shape = [(repo_url, 'WRITE'), ...]]
-        result[team_name] = list(zip(repo_urls, repo_permissions))
+        result[team_name] = [RepoPermission(url, perm) for url, perm in zip(repo_urls, repo_permissions)]
     return result
 
 
@@ -114,8 +145,8 @@ def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedG
 def transform_teams(
         team_paginated_data: PaginatedGraphqlData,
         org_data: Dict[str, Any],
-        team_repo_data: Dict[str, Any],
-) -> List[Dict[str, Any]]:
+        team_repo_data: dict[str, list[RepoPermission]],
+) -> list[dict[str, Any]]:
     result = []
     for team in team_paginated_data.nodes:
         team_name = team['slug']

{cartography-0.93.0 → cartography-0.94.0}/cartography/intel/kandji/__init__.py

@@ -21,7 +21,7 @@ def start_kandji_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
     """
     if config.kandji_base_uri is None or config.kandji_token is None or config.kandji_tenant_id is None:
         logger.warning(
-            'Required parameter(s) missing. Skipping sync.',
+            'Required parameter missing. Skipping sync. '
             'See docs to configure.',
         )
         return

{cartography-0.93.0 → cartography-0.94.0}/cartography/intel/semgrep/findings.py

@@ -3,10 +3,11 @@ from typing import Any
 from typing import Dict
 from typing import List
 from typing import Tuple
-from urllib.error import HTTPError
 
 import neo4j
 import requests
+from requests.exceptions import HTTPError
+from requests.exceptions import ReadTimeout
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
@@ -20,6 +21,7 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 stat_handler = get_stats_client(__name__)
+_PAGE_SIZE = 500
 _TIMEOUT = (60, 60)
 _MAX_RETRIES = 3
 
@@ -48,60 +50,91 @@ def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
 
 
 @timeit
-def get_sca_vulns(semgrep_app_token: str, deployment_id: str) -> List[Dict[str, Any]]:
+def get_sca_vulns(semgrep_app_token: str, deployment_slug: str) -> List[Dict[str, Any]]:
     """
     Gets the SCA vulns associated with the passed Semgrep App token and deployment id.
     param: semgrep_app_token: The Semgrep App token to use for authentication.
-    param: deployment_id: The Semgrep deployment id to use for retrieving SCA vulns.
+    param: deployment_slug: The Semgrep deployment slug to use for retrieving SCA vulns.
     """
     all_vulns = []
-    sca_url = f"https://semgrep.dev/api/v1/deployments/{deployment_id}/ssc-vulns"
+    sca_url = f"https://semgrep.dev/api/v1/deployments/{deployment_slug}/findings"
     has_more = True
-    cursor: Dict[str, str] = {}
-    page = 1
+    page = 0
     retries = 0
     headers = {
         "Content-Type": "application/json",
         "Authorization": f"Bearer {semgrep_app_token}",
     }
 
-    request_data = {
-        "deploymentId": deployment_id,
-        "pageSize": 100,
-        "exposure": ["UNREACHABLE", "REACHABLE", "UNKNOWN_EXPOSURE"],
-        "refs": ["_default"],
+    request_data: dict[str, Any] = {
+        "page": page,
+        "page_size": _PAGE_SIZE,
+        "issue_type": "sca",
+        "exposures": "reachable,always_reachable,conditionally_reachable,unreachable,unknown",
+        "ref": "_default",
+        "dedup": "true",
     }
-
+    logger.info(f"Retrieving Semgrep SCA vulns for deployment '{deployment_slug}'.")
     while has_more:
 
-        if cursor:
-            request_data.update({
-                "cursor": {
-                    "vulnOffset": cursor["vulnOffset"],
-                    "issueOffset": cursor["issueOffset"],
-                },
-            })
         try:
-            response = requests.post(sca_url, json=request_data, headers=headers, timeout=_TIMEOUT)
+            response = requests.get(sca_url, params=request_data, headers=headers, timeout=_TIMEOUT)
             response.raise_for_status()
             data = response.json()
-        except HTTPError as e:
+        except (ReadTimeout, HTTPError) as e:
             logger.warning(f"Failed to retrieve Semgrep SCA vulns for page {page}. Retrying...")
             retries += 1
             if retries >= _MAX_RETRIES:
                 raise e
             continue
-        vulns = data["vulns"]
-        cursor = data.get("cursor")
-        has_more = data.get("hasMore", False)
+        vulns = data["findings"]
+        has_more = len(vulns) > 0
         if page % 10 == 0:
-            logger.info(f"Processed {page} pages of Semgrep SCA vulnerabilities so far.")
+            logger.info(f"Processed page {page} of Semgrep SCA vulnerabilities.")
         all_vulns.extend(vulns)
         retries = 0
+        page += 1
+        request_data["page"] = page
 
+    logger.info(f"Retrieved {len(all_vulns)} Semgrep SCA vulns in {page} pages.")
     return all_vulns
 
 
+def _get_vuln_class(vuln: Dict) -> str:
+    vulnerability_classes = vuln["rule"].get("vulnerability_classes", [])
+    if vulnerability_classes:
+        return vulnerability_classes[0]
+    return "Other"
+
+
+def _determine_exposure(vuln: Dict[str, Any]) -> str | None:
+    # See Semgrep reachability types:
+    # https://semgrep.dev/docs/semgrep-supply-chain/overview#types-of-semgrep-supply-chain-findings
+    reachability_types = {
+        "NO REACHABILITY ANALYSIS": 2,
+        "UNREACHABLE": 2,
+        "REACHABLE": 0,
+        "ALWAYS REACHABLE": 0,
+        "CONDITIONALLY REACHABLE": 1,
+    }
+    reachable_flag = vuln["reachability"]
+    if reachable_flag and reachable_flag.upper() in reachability_types:
+        reach_score = reachability_types[reachable_flag.upper()]
+        if reach_score < reachability_types["UNREACHABLE"]:
+            return "REACHABLE"
+        else:
+            return "UNREACHABLE"
+    return None
+
+
+def _build_vuln_url(vuln: str) -> str | None:
+    if 'CVE' in vuln:
+        return f"https://nvd.nist.gov/vuln/detail/{vuln}"
+    if 'GHSA' in vuln:
+        return f"https://github.com/advisories/{vuln}"
+    return None
+
+
 def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
     """
     Transforms the raw SCA vulns response from Semgrep API into a list of dicts
@@ -112,46 +145,59 @@ def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str,
     for vuln in raw_vulns:
         sca_vuln: Dict[str, Any] = {}
         # Mandatory fields
-        sca_vuln["id"] = vuln["groupKey"]
-        sca_vuln["repositoryName"] = vuln["repositoryName"]
-        sca_vuln["ruleId"] = vuln["advisory"]["ruleId"]
-        sca_vuln["title"] = vuln["advisory"]["title"]
-        sca_vuln["description"] = vuln["advisory"]["description"]
-        sca_vuln["ecosystem"] = vuln["advisory"]["ecosystem"]
-        sca_vuln["severity"] = vuln["advisory"]["severity"]
-        sca_vuln["reachability"] = vuln["advisory"]["reachability"]
-        sca_vuln["reachableIf"] = vuln["advisory"]["reachableIf"]
-        sca_vuln["exposureType"] = vuln["exposureType"]
-        dependency = f"{vuln['matchedDependency']['name']}|{vuln['matchedDependency']['versionSpecifier']}"
+        repository_name = vuln["repository"]["name"]
+        rule_id = vuln["rule"]["name"]
+        vulnerability_class = _get_vuln_class(vuln)
+        package = vuln['found_dependency']['package']
+        sca_vuln["id"] = vuln["id"]
+        sca_vuln["repositoryName"] = repository_name
+        sca_vuln["branch"] = vuln["ref"]
+        sca_vuln["ruleId"] = rule_id
+        sca_vuln["title"] = package + ":" + vulnerability_class
+        sca_vuln["description"] = vuln["rule"]["message"]
+        sca_vuln["ecosystem"] = vuln["found_dependency"]["ecosystem"]
+        sca_vuln["severity"] = vuln["severity"].upper()
+        sca_vuln["reachability"] = vuln["reachability"].upper()  # Check done to determine rechabilitity
+        sca_vuln["reachableIf"] = vuln["reachable_condition"].upper() if vuln["reachable_condition"] else None
+        sca_vuln["exposureType"] = _determine_exposure(vuln)  # Determintes if reachable or unreachable
+        dependency = f"{package}|{vuln['found_dependency']['version']}"
         sca_vuln["matchedDependency"] = dependency
-        sca_vuln["dependencyFileLocation_path"] = vuln["dependencyFileLocation"]["path"]
-        sca_vuln["dependencyFileLocation_url"] = vuln["dependencyFileLocation"]["url"]
-        # Optional fields
-        sca_vuln["transitivity"] = vuln.get("transitivity", None)
-        cves = vuln.get("advisory", {}).get("references", {}).get("cveIds")
-        if len(cves) > 0:
-            # Take the first CVE
-            sca_vuln["cveId"] = vuln["advisory"]["references"]["cveIds"][0]
-        if vuln.get('closestSafeDependency'):
-            dep_fix = f"{vuln['closestSafeDependency']['name']}|{vuln['closestSafeDependency']['versionSpecifier']}"
+        dep_url = vuln["found_dependency"]["lockfile_line_url"]
+        if dep_url:  # Lock file can be null, need to set
+            dep_file = dep_url.split("/")[-1].split("#")[0]
+            sca_vuln["dependencyFileLocation_path"] = dep_file
+            sca_vuln["dependencyFileLocation_url"] = dep_url
+        else:
+            if sca_vuln.get("location"):
+                sca_vuln["dependencyFileLocation_path"] = sca_vuln["location"]["file_path"]
+        sca_vuln["transitivity"] = vuln["found_dependency"]["transitivity"].upper()
+        if vuln.get("vulnerability_identifier"):
+            vuln_id = vuln["vulnerability_identifier"].upper()
+            sca_vuln["cveId"] = vuln_id
+            sca_vuln["ref_urls"] = [_build_vuln_url(vuln_id)]
+        if vuln.get('fix_recommendations') and len(vuln['fix_recommendations']) > 0:
+            fix = vuln['fix_recommendations'][0]
+            dep_fix = f"{fix['package']}|{fix['version']}"
             sca_vuln["closestSafeDependency"] = dep_fix
-        if vuln["advisory"].get("references", {}).get("urls", []):
-            sca_vuln["ref_urls"] = vuln["advisory"].get("references", {}).get("urls", [])
-        sca_vuln["openedAt"] = vuln.get("openedAt", None)
-        sca_vuln["announcedAt"] = vuln.get("announcedAt", None)
-        sca_vuln["fixStatus"] = vuln["triage"]["status"]
-        for usage in vuln.get("usages", []):
+        sca_vuln["openedAt"] = vuln["created_at"]
+        sca_vuln["fixStatus"] = vuln["status"]
+        sca_vuln["triageStatus"] = vuln["triage_state"]
+        sca_vuln["confidence"] = vuln["confidence"]
+        usage = vuln.get("usage")
+        if usage:
             usage_dict = {}
+            url = usage["location"]["url"]
             usage_dict["SCA_ID"] = sca_vuln["id"]
-            usage_dict["findingId"] = usage["findingId"]
+            usage_dict["findingId"] = hash(url.split("github.com/")[-1])
             usage_dict["path"] = usage["location"]["path"]
-            usage_dict["startLine"] = usage["location"]["startLine"]
-            usage_dict["startCol"] = usage["location"]["startCol"]
-            usage_dict["endLine"] = usage["location"]["endLine"]
-            usage_dict["endCol"] = usage["location"]["endCol"]
-            usage_dict["url"] = usage["location"]["url"]
+            usage_dict["startLine"] = usage["location"]["start_line"]
+            usage_dict["startCol"] = usage["location"]["start_col"]
+            usage_dict["endLine"] = usage["location"]["end_line"]
+            usage_dict["endCol"] = usage["location"]["end_col"]
+            usage_dict["url"] = url
             usages.append(usage_dict)
         vulns.append(sca_vuln)
+
     return vulns, usages
 
 
@@ -228,9 +274,10 @@ def sync(
     logger.info("Running Semgrep SCA findings sync job.")
     semgrep_deployment = get_deployment(semgrep_app_token)
     deployment_id = semgrep_deployment["id"]
+    deployment_slug = semgrep_deployment["slug"]
     load_semgrep_deployment(neo4j_sesion, semgrep_deployment, update_tag)
     common_job_parameters["DEPLOYMENT_ID"] = deployment_id
-    raw_vulns = get_sca_vulns(semgrep_app_token, deployment_id)
+    raw_vulns = get_sca_vulns(semgrep_app_token, deployment_slug)
     vulns, usages = transform_sca_vulns(raw_vulns)
     load_semgrep_sca_vulns(neo4j_sesion, vulns, deployment_id, update_tag)
     load_semgrep_sca_usages(neo4j_sesion, usages, deployment_id, update_tag)

cartography-0.94.0/cartography/intel/snipeit/__init__.py

@@ -0,0 +1,30 @@
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.snipeit import asset
+from cartography.intel.snipeit import user
+from cartography.stats import get_stats_client
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_snipeit_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    if config.snipeit_base_uri is None or config.snipeit_token is None or config.snipeit_tenant_id is None:
+        logger.warning(
+            "Required parameter(s) missing. Skipping sync.",
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "TENANT_ID": config.snipeit_tenant_id,
+    }
+
+    # Ingest SnipeIT users and assets
+    user.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
+    asset.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)