cartography 0.95.0rc1__tar.gz → 0.96.0rc1__tar.gz

{cartography-0.95.0rc1 → cartography-0.96.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.95.0rc1
+Version: 0.96.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.95.0rc1 → cartography-0.96.0rc1}/README.md

@@ -80,11 +80,10 @@ Directly querying Neo4j is already very useful as a sort of "swiss army knife" f
 
 ## Community
 
-- Join us on `#cartography` on the [Lyft OSS Slack](https://join.slack.com/t/lyftoss/shared_invite/enQtOTYzODg5OTQwNDE2LTFiYjgwZWM3NTNhMTFkZjc4Y2IxOTI4NTdiNTdhNjQ4M2Q5NTIzMjVjOWI4NmVlNjRiZmU2YzA5NTc3MmFjYTQ).
+- Hang out with us on Slack: Join the CNCF Slack workspace [here](https://communityinviter.com/apps/cloud-native/cncf), and then join the `#cartography` channel.
 - Talk to us and see what we're working on at our [monthly community meeting](https://calendar.google.com/calendar/embed?src=lyft.com_p10o6ceuiieq9sqcn1ef61v1io%40group.calendar.google.com&ctz=America%2FLos_Angeles).
   - Meeting minutes are [here](https://docs.google.com/document/d/1VyRKmB0dpX185I15BmNJZpfAJ_Ooobwz0U1WIhjDxvw).
   - Recorded videos are posted [here](https://www.youtube.com/playlist?list=PLMga2YJvAGzidUWJB_fnG7EHI4wsDDsE1).
-- Our current project roadmap is [here](https://github.com/orgs/lyft/projects/26/views/1).
 
 ## License
 

{cartography-0.95.0rc1 → cartography-0.96.0rc1}/cartography/graph/querybuilder.py

@@ -118,6 +118,7 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
     """
     match = Template("$node_var.$key = $prop_ref")
     case_insensitive_match = Template("toLower($node_var.$key) = toLower($prop_ref)")
+    fuzzy_and_ignorecase_match = Template("toLower($node_var.$key) CONTAINS toLower($prop_ref)")
 
     matcher_asdict = asdict(matcher)
 
@@ -125,7 +126,10 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
     for key, prop_ref in matcher_asdict.items():
         if prop_ref.ignore_case:
             prop_line = case_insensitive_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
+        elif prop_ref.fuzzy_and_ignore_case:
+            prop_line = fuzzy_and_ignorecase_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
         else:
+            # Exact match (default; most efficient)
             prop_line = match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
         result.append(prop_line)
     return ' AND\n'.join(result)

cartography-0.96.0rc1/cartography/intel/aws/ec2/network_acls.py

@@ -0,0 +1,208 @@
+import logging
+from collections import namedtuple
+from typing import Any
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclEgressRuleSchema
+from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclInboundRuleSchema
+from cartography.models.aws.ec2.network_acls import EC2NetworkAclSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+Ec2AclObjects = namedtuple(
+    "Ec2AclObjects", [
+        'network_acls',
+        'inbound_rules',
+        'outbound_rules',
+    ],
+)
+
+
+@timeit
+@aws_handle_regions
+def get_network_acl_data(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_network_acls')
+    acls = []
+    for page in paginator.paginate():
+        acls.extend(page['NetworkAcls'])
+    return acls
+
+
+def transform_network_acl_data(
+        data_list: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+) -> Ec2AclObjects:
+    network_acls = []
+    inbound_rules = []
+    outbound_rules = []
+
+    for network_acl in data_list:
+        network_acl_id = network_acl['NetworkAclId']
+        base_network_acl = {
+            'Id': network_acl_id,
+            'Arn': f'arn:aws:ec2:{region}:{current_aws_account_id}:network-acl/{network_acl_id}',
+            'IsDefault': network_acl['IsDefault'],
+            'VpcId': network_acl['VpcId'],
+            'OwnerId': network_acl['OwnerId'],
+        }
+        if network_acl.get('Associations') and network_acl['Associations']:
+            # Include subnet associations in the data object if they exist
+            for association in network_acl['Associations']:
+                base_network_acl['NetworkAclAssociationId'] = association['NetworkAclAssociationId']
+                base_network_acl['SubnetId'] = association['SubnetId']
+                network_acls.append(base_network_acl)
+        else:
+            # Otherwise if there's no associations then don't include that in the data object
+            network_acls.append(base_network_acl)
+
+        if network_acl.get("Entries"):
+            for rule in network_acl["Entries"]:
+                direction = 'egress' if rule['Egress'] else 'inbound'
+                transformed_rule = {
+                    'Id': f"{network_acl['NetworkAclId']}/{direction}/{rule['RuleNumber']}",
+                    'CidrBlock': rule['CidrBlock'],
+                    'Egress': rule['Egress'],
+                    'Protocol': rule['Protocol'],
+                    'RuleAction': rule['RuleAction'],
+                    'RuleNumber': rule['RuleNumber'],
+                    # Add pointer back to the nacl to create an edge
+                    'NetworkAclId': network_acl_id,
+                    'FromPort': rule.get('PortRange', {}).get('FromPort'),
+                    'ToPort': rule.get('PortRange', {}).get('ToPort'),
+                }
+                if transformed_rule['Egress']:
+                    outbound_rules.append(transformed_rule)
+                else:
+                    inbound_rules.append(transformed_rule)
+    return Ec2AclObjects(
+        network_acls=network_acls,
+        inbound_rules=inbound_rules,
+        outbound_rules=outbound_rules,
+    )
+
+
+@timeit
+def load_all_nacl_data(
+        neo4j_session: neo4j.Session,
+        ec2_acl_objects: Ec2AclObjects,
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load_network_acls(
+        neo4j_session,
+        ec2_acl_objects.network_acls,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+    load_network_acl_inbound_rules(
+        neo4j_session,
+        ec2_acl_objects.inbound_rules,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+    load_network_acl_egress_rules(
+        neo4j_session,
+        ec2_acl_objects.outbound_rules,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+
+
+@timeit
+def load_network_acls(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acls in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_network_acl_inbound_rules(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acl inbound rules in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclInboundRuleSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_network_acl_egress_rules(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acl egress rules in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclEgressRuleSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup_network_acls(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    GraphJob.from_node_schema(EC2NetworkAclSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2NetworkAclInboundRuleSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2NetworkAclEgressRuleSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_network_acls(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(f"Syncing EC2 network ACLs for region '{region}' in account '{current_aws_account_id}'.")
+        data = get_network_acl_data(boto3_session, region)
+        ec2_acl_data = transform_network_acl_data(data, region, current_aws_account_id)
+        load_all_nacl_data(
+            neo4j_session,
+            ec2_acl_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        cleanup_network_acls(neo4j_session, common_job_parameters)

{cartography-0.95.0rc1 → cartography-0.96.0rc1}/cartography/intel/aws/resources.py

@@ -32,6 +32,7 @@ from .ec2.key_pairs import sync_ec2_key_pairs
 from .ec2.launch_templates import sync_ec2_launch_templates
 from .ec2.load_balancer_v2s import sync_load_balancer_v2s
 from .ec2.load_balancers import sync_load_balancers
+from .ec2.network_acls import sync_network_acls
 from .ec2.network_interfaces import sync_network_interfaces
 from .ec2.reserved_instances import sync_ec2_reserved_instances
 from .ec2.security_groups import sync_ec2_security_groupinfo
@@ -55,6 +56,7 @@ RESOURCE_FUNCTIONS: Dict = {
     'ec2:keypair': sync_ec2_key_pairs,
     'ec2:load_balancer': sync_load_balancers,
     'ec2:load_balancer_v2': sync_load_balancer_v2s,
+    'ec2:network_acls': sync_network_acls,
     'ec2:network_interface': sync_network_interfaces,
     'ec2:security_group': sync_ec2_security_groupinfo,
     'ec2:subnet': sync_subnets,

cartography-0.96.0rc1/cartography/intel/semgrep/__init__.py

@@ -0,0 +1,30 @@
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.semgrep.dependencies import sync_dependencies
+from cartography.intel.semgrep.deployment import sync_deployment
+from cartography.intel.semgrep.findings import sync_findings
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_semgrep_ingestion(
+    neo4j_session: neo4j.Session, config: Config,
+) -> None:
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+    if not config.semgrep_app_token:
+        logger.info('Semgrep import is not configured - skipping this module. See docs to configure.')
+        return
+
+    # sync_deployment must be called first since it populates common_job_parameters
+    # with the deployment ID and slug, which are required by the other sync functions
+    sync_deployment(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+    sync_dependencies(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+    sync_findings(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)

cartography-0.96.0rc1/cartography/intel/semgrep/dependencies.py

@@ -0,0 +1,201 @@
+import logging
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+from requests.exceptions import HTTPError
+from requests.exceptions import ReadTimeout
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.semgrep.dependencies import SemgrepGoLibrarySchema
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+_PAGE_SIZE = 10000
+_TIMEOUT = (60, 60)
+_MAX_RETRIES = 3
+
+
+@timeit
+def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystems: List[str]) -> List[Dict[str, Any]]:
+    """
+    Gets all dependencies for the given ecosystems within the given Semgrep deployment ID.
+    param: semgrep_app_token: The Semgrep App token to use for authentication.
+    param: deployment_id: The Semgrep deployment ID to use for retrieving dependencies.
+    param: ecosystems: One or more ecosystems to import dependencies from, e.g. "gomod" or "pypi".
+    The list of supported ecosystems is defined here:
+    https://semgrep.dev/api/v1/docs/#tag/SupplyChainService/operation/semgrep_app.products.sca.handlers.dependency.list_dependencies_conexxion
+    """
+    all_deps = []
+    deps_url = f"https://semgrep.dev/api/v1/deployments/{deployment_id}/dependencies"
+    has_more = True
+    page = 0
+    retries = 0
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {semgrep_app_token}",
+    }
+
+    request_data: dict[str, Any] = {
+        "pageSize": _PAGE_SIZE,
+        "dependencyFilter": {
+            "ecosystem": ecosystems,
+        },
+    }
+
+    logger.info(f"Retrieving Semgrep dependencies for deployment '{deployment_id}'.")
+    while has_more:
+        try:
+            response = requests.post(deps_url, json=request_data, headers=headers, timeout=_TIMEOUT)
+            response.raise_for_status()
+            data = response.json()
+        except (ReadTimeout, HTTPError):
+            logger.warning(f"Failed to retrieve Semgrep dependencies for page {page}. Retrying...")
+            retries += 1
+            if retries >= _MAX_RETRIES:
+                raise
+            continue
+        deps = data.get("dependencies", [])
+        has_more = data.get("hasMore", False)
+        logger.info(f"Processed page {page} of Semgrep dependencies.")
+        all_deps.extend(deps)
+        retries = 0
+        page += 1
+        request_data["cursor"] = data.get("cursor")
+
+    logger.info(f"Retrieved {len(all_deps)} Semgrep dependencies in {page} pages.")
+    return all_deps
+
+
+def transform_dependencies(raw_deps: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Transforms the raw dependencies response from Semgrep API into a list of dicts
+    that can be used to create the Dependency nodes.
+    """
+
+    """
+    sample raw_dep as of November 2024:
+    {
+        "repositoryId": "123456",
+        "definedAt": {
+            "path": "go.mod",
+            "startLine": "6",
+            "endLine": "6",
+            "url": "https://github.com/org/repo-name/blob/00000000000000000000000000000000/go.mod#L6",
+            "committedAt": "1970-01-01T00:00:00Z",
+            "startCol": "0",
+            "endCol": "0"
+        },
+        "transitivity": "DIRECT",
+        "package": {
+            "name": "github.com/foo/bar",
+            "versionSpecifier": "1.2.3"
+        },
+        "ecosystem": "gomod",
+        "licenses": [],
+        "pathToTransitivity": []
+    },
+    """
+    deps = []
+    for raw_dep in raw_deps:
+
+        # We could call a different endpoint to get all repo IDs and store a mapping of repo ID to URL,
+        # but it's much simpler to just extract the URL from the definedAt field.
+        repo_url = raw_dep["definedAt"]["url"].split("/blob/", 1)[0]
+
+        name = raw_dep["package"]["name"]
+        version = raw_dep["package"]["versionSpecifier"]
+        id = f"{name}|{version}"
+
+        # As of November 2024, Semgrep does not import dependencies with version specifiers such as >, <, etc.
+        # For now, hardcode the specifier to ==<version> to align with GitHub-sourced Python dependencies.
+        # If Semgrep eventually supports version specifiers, update this line accordingly.
+        specifier = f"=={version}"
+
+        deps.append({
+            # existing dependency properties:
+            "id": id,
+            "name": name,
+            "specifier": specifier,
+            "version": version,
+            "repo_url": repo_url,
+
+            # Semgrep-specific properties:
+            "ecosystem": raw_dep["ecosystem"],
+            "transitivity": raw_dep["transitivity"].lower(),
+            "url": raw_dep["definedAt"]["url"],
+        })
+
+    return deps
+
+
+@timeit
+def load_dependencies(
+    neo4j_session: neo4j.Session,
+    dependency_schema: Callable,
+    dependencies: List[Dict],
+    deployment_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(dependencies)} {dependency_schema().label} objects into the graph.")
+    load(
+        neo4j_session,
+        dependency_schema(),
+        dependencies,
+        lastupdated=update_tag,
+        DEPLOYMENT_ID=deployment_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.info("Running Semgrep Go Library cleanup job.")
+    go_libraries_cleanup_job = GraphJob.from_node_schema(
+        SemgrepGoLibrarySchema(), common_job_parameters,
+    )
+    go_libraries_cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync_dependencies(
+    neo4j_session: neo4j.Session,
+    semgrep_app_token: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+
+    deployment_id = common_job_parameters.get("DEPLOYMENT_ID")
+    if not deployment_id:
+        logger.warning(
+            "Missing Semgrep deployment ID, ensure that sync_deployment() has been called."
+            "Skipping Semgrep dependencies sync job.",
+        )
+        return
+
+    logger.info("Running Semgrep dependencies sync job.")
+
+    # fetch and load dependencies for the Go ecosystem
+    raw_go_deps = get_dependencies(semgrep_app_token, deployment_id, ecosystems=["gomod"])
+    go_deps = transform_dependencies(raw_go_deps)
+    load_dependencies(neo4j_session, SemgrepGoLibrarySchema, go_deps, deployment_id, update_tag)
+
+    cleanup(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session=neo4j_session,
+        group_type='Semgrep',
+        group_id=deployment_id,
+        synced_type='SemgrepDependency',
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography-0.96.0rc1/cartography/intel/semgrep/deployment.py

@@ -0,0 +1,67 @@
+import logging
+from typing import Any
+from typing import Dict
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.models.semgrep.deployment import SemgrepDeploymentSchema
+from cartography.stats import get_stats_client
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
+    """
+    Gets the deployment associated with the passed Semgrep App token.
+    param: semgrep_app_token: The Semgrep App token to use for authentication.
+    """
+    deployment = {}
+    deployment_url = "https://semgrep.dev/api/v1/deployments"
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {semgrep_app_token}",
+    }
+    response = requests.get(deployment_url, headers=headers, timeout=_TIMEOUT)
+    response.raise_for_status()
+
+    data = response.json()
+    deployment["id"] = data["deployments"][0]["id"]
+    deployment["name"] = data["deployments"][0]["name"]
+    deployment["slug"] = data["deployments"][0]["slug"]
+
+    return deployment
+
+
+@timeit
+def load_semgrep_deployment(
+    neo4j_session: neo4j.Session, deployment: Dict[str, Any], update_tag: int,
+) -> None:
+    logger.info(f"Loading SemgrepDeployment {deployment} into the graph.")
+    load(
+        neo4j_session,
+        SemgrepDeploymentSchema(),
+        [deployment],
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def sync_deployment(
+    neo4j_session: neo4j.Session,
+    semgrep_app_token: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+
+    semgrep_deployment = get_deployment(semgrep_app_token)
+    deployment_id = semgrep_deployment["id"]
+    deployment_slug = semgrep_deployment["slug"]
+    load_semgrep_deployment(neo4j_session, semgrep_deployment, update_tag)
+    common_job_parameters["DEPLOYMENT_ID"] = deployment_id
+    common_job_parameters["DEPLOYMENT_SLUG"] = deployment_slug