cartography 0.107.0rc1__tar.gz → 0.107.0rc2__tar.gz

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.107.0rc1
+Version: 0.107.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.107.0rc1'
-__version_tuple__ = version_tuple = (0, 107, 0, 'rc1')
+__version__ = version = '0.107.0rc2'
+__version_tuple__ = version_tuple = (0, 107, 0, 'rc2')

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/cartography/intel/aws/inspector.py

@@ -3,14 +3,17 @@ from typing import Any
 from typing import Dict
 from typing import Iterator
 from typing import List
+from typing import Set
 from typing import Tuple
 
 import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
+from cartography.models.aws.inspector.findings import InspectorFindingToPackageMatchLink
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
@@ -107,9 +110,10 @@ def get_inspector_findings(
 
 def transform_inspector_findings(
     results: List[Dict[str, Any]],
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, str]]]:
     findings_list: List[Dict] = []
-    packages: Dict[str, Any] = {}
+    packages_set: Set[frozenset] = set()
+    finding_to_package_map: List[Dict[str, str]] = []
 
     for f in results:
         finding: Dict = {}
@@ -163,55 +167,45 @@ def transform_inspector_findings(
                 "vendorUpdatedAt",
             )
 
-            new_packages = _process_packages(
-                f["packageVulnerabilityDetails"],
-                f["awsAccountId"],
-                f["findingArn"],
-            )
-            finding["vulnerablepackageids"] = list(new_packages.keys())
-            packages = {**packages, **new_packages}
-
+            packages = transform_inspector_packages(f["packageVulnerabilityDetails"])
+            finding["vulnerablepackageids"] = list(packages.keys())
+            for package_id, package in packages.items():
+                finding_to_package_map.append(
+                    {
+                        "findingarn": finding["id"],
+                        "packageid": package_id,
+                        "remediation": package.get("remediation"),
+                        "fixedInVersion": package.get("fixedInVersion"),
+                        "filePath": package.get("filePath"),
+                        "sourceLayerHash": package.get("sourceLayerHash"),
+                        "sourceLambdaLayerArn": package.get("sourceLambdaLayerArn"),
+                    }
+                )
+                packages_set.add(frozenset(package.items()))
         findings_list.append(finding)
-    packages_list = transform_inspector_packages(packages)
-    return findings_list, packages_list
-
+    packages_list = [dict(p) for p in packages_set]
+    return findings_list, packages_list, finding_to_package_map
 
-def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict[str, Any]]:
-    packages_list: List[Dict] = []
-    for package_id in packages.keys():
-        packages_list.append(packages[package_id])
 
-    return packages_list
-
-
-def _process_packages(
+def transform_inspector_packages(
     package_details: Dict[str, Any],
-    aws_account_id: str,
-    finding_arn: str,
 ) -> Dict[str, Any]:
     packages: Dict[str, Any] = {}
     for package in package_details["vulnerablePackages"]:
-        new_package = {}
-        new_package["id"] = (
-            f"{package.get('name', '')}|"
-            f"{package.get('arch', '')}|"
-            f"{package.get('version', '')}|"
-            f"{package.get('release', '')}|"
-            f"{package.get('epoch', '')}"
-        )
-        new_package["name"] = package.get("name")
-        new_package["arch"] = package.get("arch")
-        new_package["version"] = package.get("version")
-        new_package["release"] = package.get("release")
-        new_package["epoch"] = package.get("epoch")
-        new_package["manager"] = package.get("packageManager")
-        new_package["filepath"] = package.get("filePath")
-        new_package["fixedinversion"] = package.get("fixedInVersion")
-        new_package["sourcelayerhash"] = package.get("sourceLayerHash")
-        new_package["awsaccount"] = aws_account_id
-        new_package["findingarn"] = finding_arn
-
-        packages[new_package["id"]] = new_package
+        # Following RPM package naming convention for consistency
+        name = package["name"]  # Mandatory field
+        epoch = str(package.get("epoch", ""))
+        if epoch:
+            epoch = f"{epoch}:"
+        version = package["version"]  # Mandatory field
+        release = package.get("release", "")
+        if release:
+            release = f"-{release}"
+        arch = package.get("arch", "")
+        if arch:
+            arch = f".{arch}"
+        id = f"{name}|{epoch}{version}{release}{arch}"
+        packages[id] = {**package, "id": id}
 
     return packages
 
@@ -244,7 +238,6 @@ def load_inspector_findings(
 def load_inspector_packages(
     neo4j_session: neo4j.Session,
     packages: List[Dict[str, Any]],
-    region: str,
     aws_update_tag: int,
     current_aws_account_id: str,
 ) -> None:
@@ -252,12 +245,28 @@ def load_inspector_packages(
         neo4j_session,
         AWSInspectorPackageSchema(),
         packages,
-        Region=region,
         AWS_ID=current_aws_account_id,
         lastupdated=aws_update_tag,
     )
 
 
+@timeit
+def load_inspector_finding_to_package_match_links(
+    neo4j_session: neo4j.Session,
+    finding_to_package_map: List[Dict[str, str]],
+    aws_update_tag: int,
+    current_aws_account_id: str,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        InspectorFindingToPackageMatchLink(),
+        finding_to_package_map,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -270,6 +279,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_matchlink(
+        InspectorFindingToPackageMatchLink(),
+        "AWSAccount",
+        common_job_parameters["ACCOUNT_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(
+        neo4j_session,
+    )
 
 
 def _sync_findings_for_account(
@@ -288,7 +305,9 @@ def _sync_findings_for_account(
         logger.info(f"No findings to sync for account {account_id} in region {region}")
         return
     for f_batch in findings:
-        finding_data, package_data = transform_inspector_findings(f_batch)
+        finding_data, package_data, finding_to_package_map = (
+            transform_inspector_findings(f_batch)
+        )
         logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
         load_inspector_findings(
             neo4j_session,
@@ -301,7 +320,15 @@ def _sync_findings_for_account(
         load_inspector_packages(
             neo4j_session,
             package_data,
-            region,
+            update_tag,
+            current_aws_account_id,
+        )
+        logger.info(
+            f"Loading {len(finding_to_package_map)} finding to package relationships"
+        )
+        load_inspector_finding_to_package_match_links(
+            neo4j_session,
+            finding_to_package_map,
             update_tag,
             current_aws_account_id,
         )
@@ -337,5 +364,7 @@ def sync(
                 update_tag,
                 current_aws_account_id,
             )
+        common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
+        common_job_parameters["UPDATE_TAG"] = update_tag
 
     cleanup(neo4j_session, common_job_parameters)

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/cartography/models/aws/inspector/findings.py

@@ -7,8 +7,10 @@ from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_source_node_matcher
 from cartography.models.core.relationships import make_target_node_matcher
 from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import SourceNodeMatcher
 from cartography.models.core.relationships import TargetNodeMatcher
 
 
@@ -135,6 +137,40 @@ class InspectorFindingToECRImageRel(CartographyRelSchema):
     )
 
 
+@dataclass(frozen=True)
+class InspectorFindingToPackageRelRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+    _sub_resource_label: PropertyRef = PropertyRef(
+        "_sub_resource_label", set_in_kwargs=True
+    )
+    _sub_resource_id: PropertyRef = PropertyRef("_sub_resource_id", set_in_kwargs=True)
+    # The following properties live in vulnerablePackages from AWS API
+    # Adding them here to avoid multiple repetion of packages
+    filepath: PropertyRef = PropertyRef("filePath")
+    fixedinversion: PropertyRef = PropertyRef("fixedInVersion")
+    remediation: PropertyRef = PropertyRef("remediation")
+    sourcelayerhash: PropertyRef = PropertyRef("sourceLayerHash")
+    sourcelambdalayerarn: PropertyRef = PropertyRef("sourceLambdaLayerArn")
+
+
+@dataclass(frozen=True)
+# (:AWSInspectorFinding)-[:HAS]->(:AWSInspectorPackage)
+class InspectorFindingToPackageMatchLink(CartographyRelSchema):
+    target_node_label: str = "AWSInspectorPackage"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("packageid")},
+    )
+    source_node_label: str = "AWSInspectorFinding"
+    source_node_matcher: SourceNodeMatcher = make_source_node_matcher(
+        {"id": PropertyRef("findingarn")},
+    )
+    properties: InspectorFindingToPackageRelRelProperties = (
+        InspectorFindingToPackageRelRelProperties()
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "HAS"
+
+
 @dataclass(frozen=True)
 class AWSInspectorFindingSchema(CartographyNodeSchema):
     label: str = "AWSInspectorFinding"
@@ -146,6 +182,7 @@ class AWSInspectorFindingSchema(CartographyNodeSchema):
     other_relationships: OtherRelationships = OtherRelationships(
         [
             InspectorFindingToEC2InstanceRel(),
+            # TODO: Fix ECRRepository and ECRImage relationships
             InspectorFindingToECRRepositoryRel(),
             InspectorFindingToECRImageRel(),
             InspectorFindingToAWSAccountRelDelegateRel(),

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/cartography/models/aws/inspector/packages.py

@@ -7,25 +7,18 @@ from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
 from cartography.models.core.relationships import make_target_node_matcher
-from cartography.models.core.relationships import OtherRelationships
 from cartography.models.core.relationships import TargetNodeMatcher
 
 
 @dataclass(frozen=True)
 class AWSInspectorPackageNodeProperties(CartographyNodeProperties):
     id: PropertyRef = PropertyRef("id")
-    region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
-    awsaccount: PropertyRef = PropertyRef("AWS_ID", set_in_kwargs=True)
-    findingarn: PropertyRef = PropertyRef("findingarn", extra_index=True)
     name: PropertyRef = PropertyRef("name", extra_index=True)
-    arch: PropertyRef = PropertyRef("arch")
     version: PropertyRef = PropertyRef("version", extra_index=True)
     release: PropertyRef = PropertyRef("release", extra_index=True)
+    arch: PropertyRef = PropertyRef("arch")
     epoch: PropertyRef = PropertyRef("epoch")
     manager: PropertyRef = PropertyRef("packageManager")
-    filepath: PropertyRef = PropertyRef("filePath")
-    fixedinversion: PropertyRef = PropertyRef("fixedInVersion")
-    sourcelayerhash: PropertyRef = PropertyRef("sourceLayerHash")
     lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
 
@@ -47,24 +40,6 @@ class InspectorPackageToAWSAccountRel(CartographyRelSchema):
     )
 
 
-@dataclass(frozen=True)
-class InspectorPackageToFindingRelRelProperties(CartographyRelProperties):
-    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
-
-
-@dataclass(frozen=True)
-class InspectorPackageToFindingRel(CartographyRelSchema):
-    target_node_label: str = "AWSInspectorFinding"
-    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {"id": PropertyRef("findingarn")},
-    )
-    direction: LinkDirection = LinkDirection.INWARD
-    rel_label: str = "HAS"
-    properties: InspectorPackageToFindingRelRelProperties = (
-        InspectorPackageToFindingRelRelProperties()
-    )
-
-
 @dataclass(frozen=True)
 class AWSInspectorPackageSchema(CartographyNodeSchema):
     label: str = "AWSInspectorPackage"
@@ -72,8 +47,3 @@ class AWSInspectorPackageSchema(CartographyNodeSchema):
     sub_resource_relationship: InspectorPackageToAWSAccountRel = (
         InspectorPackageToAWSAccountRel()
     )
-    other_relationships: OtherRelationships = OtherRelationships(
-        [
-            InspectorPackageToFindingRel(),
-        ],
-    )

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/cartography.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.107.0rc1
+Version: 0.107.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/docs/root/dev/matchlinks.md

@@ -183,3 +183,99 @@ if __name__ == "__main__":
         cleanup_job = GraphJob.from_matchlink(S3AccessMatchLink(), "AWSAccount", ACCOUNT_ID, UPDATE_TAG)
         cleanup_job.run(neo4j_session)
 ```
+
+## Example 2: Adding Extended Properties to Relationships
+
+This example shows how to use MatchLinks to add rich properties to relationships between nodes. We'll use AWS Inspector findings and packages as an example, where the relationship includes important metadata like remediation information, fixed versions, and file paths.
+
+1. Define the mapping data with properties
+```python
+finding_to_package_mapping = [
+    {
+        "findingarn": "arn:aws:inspector2:us-east-1:123456789012:finding/abc123",
+        "packageid": "openssl|0:1.1.1k-1.el8.x86_64",
+        "filePath": "/usr/lib64/libssl.so.1.1",
+        "fixedInVersion": "0:1.1.1l-1.el8",
+        "remediation": "Update OpenSSL to version 1.1.1l or later",
+        "sourceLayerHash": "sha256:abc123...",
+        "sourceLambdaLayerArn": "arn:aws:lambda:us-east-1:123456789012:layer:my-layer:1",
+    },
+    {
+        "findingarn": "arn:aws:inspector2:us-east-1:123456789012:finding/def456",
+        "packageid": "openssl|0:1.1.1k-1.el8.x86_64",
+        "filePath": "/usr/lib64/libssl.so.1.1",
+        "fixedInVersion": "0:1.1.1l-1.el8",
+        "remediation": "Update OpenSSL to version 1.1.1l or later",
+        "sourceLayerHash": "sha256:abc123...",
+        "sourceLambdaLayerArn": None,
+    },
+    {
+        "findingarn": "arn:aws:inspector2:us-east-1:123456789012:finding/abc123",
+        "packageid": "curl|7.61.1-12.el8.x86_64",
+        "filePath": "/usr/bin/curl",
+        "fixedInVersion": "7.61.1-14.el8",
+        "remediation": "Update curl to version 7.61.1-14.el8 or later",
+        "sourceLayerHash": None,
+        "sourceLambdaLayerArn": None,
+    }
+]
+```
+
+1. Define the relationship properties with multiple fields
+```python
+@dataclass(frozen=True)
+class InspectorFindingToPackageRelProperties(CartographyRelProperties):
+    # Mandatory fields for MatchLinks
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+    _sub_resource_label: PropertyRef = PropertyRef("_sub_resource_label", set_in_kwargs=True)
+    _sub_resource_id: PropertyRef = PropertyRef("_sub_resource_id", set_in_kwargs=True)
+
+    # Business properties from the vulnerable package data
+    filepath: PropertyRef = PropertyRef("filePath")
+    fixedinversion: PropertyRef = PropertyRef("fixedInVersion")
+    remediation: PropertyRef = PropertyRef("remediation")
+    sourcelayerhash: PropertyRef = PropertyRef("sourceLayerHash")
+    sourcelambdalayerarn: PropertyRef = PropertyRef("sourceLambdaLayerArn")
+```
+
+1. Define the MatchLink relationship schema
+```python
+@dataclass(frozen=True)
+class InspectorFindingToPackageMatchLink(CartographyRelSchema):
+    target_node_label: str = "AWSInspectorPackage"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("packageid")},
+    )
+    source_node_label: str = "AWSInspectorFinding"
+    source_node_matcher: SourceNodeMatcher = make_source_node_matcher(
+        {"id": PropertyRef("findingarn")},
+    )
+    properties: InspectorFindingToPackageRelProperties = (
+        InspectorFindingToPackageRelProperties()
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "HAS_VULNERABLE_PACKAGE"
+```
+
+1. Load the matchlinks with properties
+```python
+load_matchlinks(
+    neo4j_session,
+    InspectorFindingToPackageMatchLink(),
+    finding_to_package_mapping,
+    lastupdated=update_tag,
+    _sub_resource_label="AWSAccount",
+    _sub_resource_id=account_id,
+)
+```
+
+1. Cleanup stale relationships
+```python
+cleanup_job = GraphJob.from_matchlink(
+    InspectorFindingToPackageMatchLink(),
+    "AWSAccount", # _sub_resource_label
+    account_id, # _sub_resource_id
+    update_tag,
+)
+cleanup_job.run(neo4j_session)
+```

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/docs/root/modules/aws/schema.md

@@ -213,19 +213,13 @@ Representation of an AWS [Inspector Finding Package](https://docs.aws.amazon.com
 | Field | Description | Required|
 |-------|-------------|------|
 |**arn**|The AWS ARN|yes|
-|id|Uses the format of `name\|arch\|version\|release\|epoch` to uniqulely identify packages|yes|
-|region|AWS region the finding is from|yes|
-|awsaccount|AWS account the finding is from|yes|
-|findingarn|The AWS ARN for a related finding|yes|
+|id|Uses the format of `name|epoch:version-release.arch` to uniquely identify packages|yes|
 |name|The finding name|
 |arch|Architecture for the package|
 |version|Version of the package|
 |release|Release of the package
 |epoch|Package epoch|
 |manager|Related package manager|
-|filepath|Path to the file or package|
-|fixedinversion|Version the related finding was fixed in|
-|sourcelayerhash|Source layer hash for container images|
 
 
 #### Relationships
@@ -234,7 +228,20 @@ Representation of an AWS [Inspector Finding Package](https://docs.aws.amazon.com
 
     ```cypher
     (:AWSInspectorFindings)-[:HAS]->(:AWSInspectorPackages)
+
     ```
+    - `HAS` attributes
+
+| Field | Description | Required|
+|-------|-------------|------|
+|filepath|Path to the file or package|
+|sourcelayerhash|Source layer hash for container images|
+|sourcelambdalayerarn|ARN of the AWS Lambda function affected|
+|fixedinversion|Version the related finding was fixed in|
+|remediation|Remediation steps|
+|_sub_resource_label|Resource label to do relationships clean-up. Always `AWSAccount`
+|_sub_resource_id|Resource id to do relationships clean-up. Always ID of the AWS `RESOURCE` account.
+
 
 - AWSInspectorPackages belong to AWSAccounts.
 

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/tests/data/aws/inspector.py

@@ -151,6 +151,8 @@ LIST_FINDINGS_EC2_PACKAGE = [
                     "packageManager": "OS",
                     "release": "6.29.amzn1",
                     "version": "4.9.17",
+                    "fixedInVersion": "0:4.9.18-6.30.amzn1.X86_64",
+                    "remediation": "Upgrade your installed software packages to the proposed fixed in version and release.\n\nyum update kernel\n\nyum update kernel-tools",
                 },
                 {
                     "arch": "X86_64",
@@ -159,6 +161,8 @@ LIST_FINDINGS_EC2_PACKAGE = [
                     "packageManager": "OS",
                     "release": "6.29.amzn1",
                     "version": "4.9.17",
+                    "fixedInVersion": "0:4.9.18-6.30.amzn1.X86_64",
+                    "remediation": "Upgrade your installed software packages to the proposed fixed in version and release.\n\nyum update kernel\n\nyum update kernel-tools",
                 },
             ],
         },

{cartography-0.107.0rc1 → cartography-0.107.0rc2}/tests/integration/cartography/intel/aws/inspector/test_inspector_sync.py

@@ -134,9 +134,9 @@ def test_sync_inspector_ec2_package_findings(mock_get, neo4j_session):
         "HAS",
         rel_direction_right=True,
     ) == {
-        ("arn:aws:test456", "kernel-tools|X86_64|4.9.17|6.29.amzn1|0"),
-        ("arn:aws:test456", "kernel|X86_64|4.9.17|6.29.amzn1|0"),
-        ("arn:aws:test789", "openssl|X86_64|1.0.2k|1.amzn2|0"),
+        ("arn:aws:test456", "kernel-tools|0:4.9.17-6.29.amzn1.X86_64"),
+        ("arn:aws:test456", "kernel|0:4.9.17-6.29.amzn1.X86_64"),
+        ("arn:aws:test789", "openssl|0:1.0.2k-1.amzn2.X86_64"),
     }
 
     # Assert AWSAccount RESOURCE to Finding exists
@@ -166,3 +166,18 @@ def test_sync_inspector_ec2_package_findings(mock_get, neo4j_session):
         ("123456789011", "arn:aws:test789"),
         ("123456789012", "arn:aws:test456"),
     }
+
+    # Assert AWSAccount RESOURCE to Package exists
+    assert check_rels(
+        neo4j_session,
+        "AWSAccount",
+        "id",
+        "AWSInspectorPackage",
+        "id",
+        "RESOURCE",
+        rel_direction_right=True,
+    ) == {
+        ("123456789012", "kernel-tools|0:4.9.17-6.29.amzn1.X86_64"),
+        ("123456789012", "kernel|0:4.9.17-6.29.amzn1.X86_64"),
+        ("123456789012", "openssl|0:1.0.2k-1.amzn2.X86_64"),
+    }