cartography 0.107.0rc1__py3-none-any.whl → 0.107.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.107.0rc1'
-__version_tuple__ = version_tuple = (0, 107, 0, 'rc1')
+__version__ = version = '0.107.0rc2'
+__version_tuple__ = version_tuple = (0, 107, 0, 'rc2')

cartography/intel/aws/inspector.py

@@ -3,14 +3,17 @@ from typing import Any
 from typing import Dict
 from typing import Iterator
 from typing import List
+from typing import Set
 from typing import Tuple
 
 import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
+from cartography.models.aws.inspector.findings import InspectorFindingToPackageMatchLink
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
@@ -107,9 +110,10 @@ def get_inspector_findings(
 
 def transform_inspector_findings(
     results: List[Dict[str, Any]],
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, str]]]:
     findings_list: List[Dict] = []
-    packages: Dict[str, Any] = {}
+    packages_set: Set[frozenset] = set()
+    finding_to_package_map: List[Dict[str, str]] = []
 
     for f in results:
         finding: Dict = {}
@@ -163,55 +167,45 @@ def transform_inspector_findings(
                 "vendorUpdatedAt",
             )
 
-            new_packages = _process_packages(
-                f["packageVulnerabilityDetails"],
-                f["awsAccountId"],
-                f["findingArn"],
-            )
-            finding["vulnerablepackageids"] = list(new_packages.keys())
-            packages = {**packages, **new_packages}
-
+            packages = transform_inspector_packages(f["packageVulnerabilityDetails"])
+            finding["vulnerablepackageids"] = list(packages.keys())
+            for package_id, package in packages.items():
+                finding_to_package_map.append(
+                    {
+                        "findingarn": finding["id"],
+                        "packageid": package_id,
+                        "remediation": package.get("remediation"),
+                        "fixedInVersion": package.get("fixedInVersion"),
+                        "filePath": package.get("filePath"),
+                        "sourceLayerHash": package.get("sourceLayerHash"),
+                        "sourceLambdaLayerArn": package.get("sourceLambdaLayerArn"),
+                    }
+                )
+                packages_set.add(frozenset(package.items()))
         findings_list.append(finding)
-    packages_list = transform_inspector_packages(packages)
-    return findings_list, packages_list
-
+    packages_list = [dict(p) for p in packages_set]
+    return findings_list, packages_list, finding_to_package_map
 
-def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict[str, Any]]:
-    packages_list: List[Dict] = []
-    for package_id in packages.keys():
-        packages_list.append(packages[package_id])
 
-    return packages_list
-
-
-def _process_packages(
+def transform_inspector_packages(
     package_details: Dict[str, Any],
-    aws_account_id: str,
-    finding_arn: str,
 ) -> Dict[str, Any]:
     packages: Dict[str, Any] = {}
     for package in package_details["vulnerablePackages"]:
-        new_package = {}
-        new_package["id"] = (
-            f"{package.get('name', '')}|"
-            f"{package.get('arch', '')}|"
-            f"{package.get('version', '')}|"
-            f"{package.get('release', '')}|"
-            f"{package.get('epoch', '')}"
-        )
-        new_package["name"] = package.get("name")
-        new_package["arch"] = package.get("arch")
-        new_package["version"] = package.get("version")
-        new_package["release"] = package.get("release")
-        new_package["epoch"] = package.get("epoch")
-        new_package["manager"] = package.get("packageManager")
-        new_package["filepath"] = package.get("filePath")
-        new_package["fixedinversion"] = package.get("fixedInVersion")
-        new_package["sourcelayerhash"] = package.get("sourceLayerHash")
-        new_package["awsaccount"] = aws_account_id
-        new_package["findingarn"] = finding_arn
-
-        packages[new_package["id"]] = new_package
+        # Following RPM package naming convention for consistency
+        name = package["name"]  # Mandatory field
+        epoch = str(package.get("epoch", ""))
+        if epoch:
+            epoch = f"{epoch}:"
+        version = package["version"]  # Mandatory field
+        release = package.get("release", "")
+        if release:
+            release = f"-{release}"
+        arch = package.get("arch", "")
+        if arch:
+            arch = f".{arch}"
+        id = f"{name}|{epoch}{version}{release}{arch}"
+        packages[id] = {**package, "id": id}
 
     return packages
 
@@ -244,7 +238,6 @@ def load_inspector_findings(
 def load_inspector_packages(
     neo4j_session: neo4j.Session,
     packages: List[Dict[str, Any]],
-    region: str,
     aws_update_tag: int,
     current_aws_account_id: str,
 ) -> None:
@@ -252,12 +245,28 @@ def load_inspector_packages(
         neo4j_session,
         AWSInspectorPackageSchema(),
         packages,
-        Region=region,
         AWS_ID=current_aws_account_id,
         lastupdated=aws_update_tag,
     )
 
 
+@timeit
+def load_inspector_finding_to_package_match_links(
+    neo4j_session: neo4j.Session,
+    finding_to_package_map: List[Dict[str, str]],
+    aws_update_tag: int,
+    current_aws_account_id: str,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        InspectorFindingToPackageMatchLink(),
+        finding_to_package_map,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -270,6 +279,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_matchlink(
+        InspectorFindingToPackageMatchLink(),
+        "AWSAccount",
+        common_job_parameters["ACCOUNT_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(
+        neo4j_session,
+    )
 
 
 def _sync_findings_for_account(
@@ -288,7 +305,9 @@ def _sync_findings_for_account(
         logger.info(f"No findings to sync for account {account_id} in region {region}")
         return
     for f_batch in findings:
-        finding_data, package_data = transform_inspector_findings(f_batch)
+        finding_data, package_data, finding_to_package_map = (
+            transform_inspector_findings(f_batch)
+        )
         logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
         load_inspector_findings(
             neo4j_session,
@@ -301,7 +320,15 @@ def _sync_findings_for_account(
         load_inspector_packages(
             neo4j_session,
             package_data,
-            region,
+            update_tag,
+            current_aws_account_id,
+        )
+        logger.info(
+            f"Loading {len(finding_to_package_map)} finding to package relationships"
+        )
+        load_inspector_finding_to_package_match_links(
+            neo4j_session,
+            finding_to_package_map,
             update_tag,
             current_aws_account_id,
         )
@@ -337,5 +364,7 @@ def sync(
                 update_tag,
                 current_aws_account_id,
             )
+        common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
+        common_job_parameters["UPDATE_TAG"] = update_tag
 
     cleanup(neo4j_session, common_job_parameters)

cartography/models/aws/inspector/findings.py

@@ -7,8 +7,10 @@ from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_source_node_matcher
 from cartography.models.core.relationships import make_target_node_matcher
 from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import SourceNodeMatcher
 from cartography.models.core.relationships import TargetNodeMatcher
 
 
@@ -135,6 +137,40 @@ class InspectorFindingToECRImageRel(CartographyRelSchema):
     )
 
 
+@dataclass(frozen=True)
+class InspectorFindingToPackageRelRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+    _sub_resource_label: PropertyRef = PropertyRef(
+        "_sub_resource_label", set_in_kwargs=True
+    )
+    _sub_resource_id: PropertyRef = PropertyRef("_sub_resource_id", set_in_kwargs=True)
+    # The following properties live in vulnerablePackages from AWS API
+    # Adding them here to avoid multiple repetion of packages
+    filepath: PropertyRef = PropertyRef("filePath")
+    fixedinversion: PropertyRef = PropertyRef("fixedInVersion")
+    remediation: PropertyRef = PropertyRef("remediation")
+    sourcelayerhash: PropertyRef = PropertyRef("sourceLayerHash")
+    sourcelambdalayerarn: PropertyRef = PropertyRef("sourceLambdaLayerArn")
+
+
+@dataclass(frozen=True)
+# (:AWSInspectorFinding)-[:HAS]->(:AWSInspectorPackage)
+class InspectorFindingToPackageMatchLink(CartographyRelSchema):
+    target_node_label: str = "AWSInspectorPackage"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("packageid")},
+    )
+    source_node_label: str = "AWSInspectorFinding"
+    source_node_matcher: SourceNodeMatcher = make_source_node_matcher(
+        {"id": PropertyRef("findingarn")},
+    )
+    properties: InspectorFindingToPackageRelRelProperties = (
+        InspectorFindingToPackageRelRelProperties()
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "HAS"
+
+
 @dataclass(frozen=True)
 class AWSInspectorFindingSchema(CartographyNodeSchema):
     label: str = "AWSInspectorFinding"
@@ -146,6 +182,7 @@ class AWSInspectorFindingSchema(CartographyNodeSchema):
     other_relationships: OtherRelationships = OtherRelationships(
         [
             InspectorFindingToEC2InstanceRel(),
+            # TODO: Fix ECRRepository and ECRImage relationships
             InspectorFindingToECRRepositoryRel(),
             InspectorFindingToECRImageRel(),
             InspectorFindingToAWSAccountRelDelegateRel(),

cartography/models/aws/inspector/packages.py

@@ -7,25 +7,18 @@ from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
 from cartography.models.core.relationships import make_target_node_matcher
-from cartography.models.core.relationships import OtherRelationships
 from cartography.models.core.relationships import TargetNodeMatcher
 
 
 @dataclass(frozen=True)
 class AWSInspectorPackageNodeProperties(CartographyNodeProperties):
     id: PropertyRef = PropertyRef("id")
-    region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
-    awsaccount: PropertyRef = PropertyRef("AWS_ID", set_in_kwargs=True)
-    findingarn: PropertyRef = PropertyRef("findingarn", extra_index=True)
     name: PropertyRef = PropertyRef("name", extra_index=True)
-    arch: PropertyRef = PropertyRef("arch")
     version: PropertyRef = PropertyRef("version", extra_index=True)
     release: PropertyRef = PropertyRef("release", extra_index=True)
+    arch: PropertyRef = PropertyRef("arch")
     epoch: PropertyRef = PropertyRef("epoch")
     manager: PropertyRef = PropertyRef("packageManager")
-    filepath: PropertyRef = PropertyRef("filePath")
-    fixedinversion: PropertyRef = PropertyRef("fixedInVersion")
-    sourcelayerhash: PropertyRef = PropertyRef("sourceLayerHash")
     lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
 
@@ -47,24 +40,6 @@ class InspectorPackageToAWSAccountRel(CartographyRelSchema):
     )
 
 
-@dataclass(frozen=True)
-class InspectorPackageToFindingRelRelProperties(CartographyRelProperties):
-    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
-
-
-@dataclass(frozen=True)
-class InspectorPackageToFindingRel(CartographyRelSchema):
-    target_node_label: str = "AWSInspectorFinding"
-    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {"id": PropertyRef("findingarn")},
-    )
-    direction: LinkDirection = LinkDirection.INWARD
-    rel_label: str = "HAS"
-    properties: InspectorPackageToFindingRelRelProperties = (
-        InspectorPackageToFindingRelRelProperties()
-    )
-
-
 @dataclass(frozen=True)
 class AWSInspectorPackageSchema(CartographyNodeSchema):
     label: str = "AWSInspectorPackage"
@@ -72,8 +47,3 @@ class AWSInspectorPackageSchema(CartographyNodeSchema):
     sub_resource_relationship: InspectorPackageToAWSAccountRel = (
         InspectorPackageToAWSAccountRel()
     )
-    other_relationships: OtherRelationships = OtherRelationships(
-        [
-            InspectorPackageToFindingRel(),
-        ],
-    )

{cartography-0.107.0rc1.dist-info → cartography-0.107.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.107.0rc1
+Version: 0.107.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2

{cartography-0.107.0rc1.dist-info → cartography-0.107.0rc2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cartography/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/__main__.py,sha256=y5iqUrj4BmqZfjeDT-9balzpXeMARgHeIedRMRI1gr8,100
-cartography/_version.py,sha256=p-Djna7oRUSPYcKo49GYVaSUJK3PW0h7Jw79Ngeb9Mo,525
+cartography/_version.py,sha256=9t4xjY9Ix9CT_vuPkt5BMT5Cw71TXl8xuRRWDmRjqAg,525
 cartography/cli.py,sha256=e5F9239v_JoLR293JMZI1toYg43li2RX_F-qOHTB-HA,46054
 cartography/config.py,sha256=mM7Frg8maGB4a0Oad2nvktM38uC7zgrwSmofO5l4Aus,16492
 cartography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -172,7 +172,7 @@ cartography/intel/aws/emr.py,sha256=EJoKjHQP7-F_A1trUNU05Sb42yNR1i0C9VIpGcCfAXw,
 cartography/intel/aws/iam.py,sha256=bkyIzWw2OC4MHxuoCvTiZ5eEGEQhz2asiUgY_tkX2GY,34322
 cartography/intel/aws/iam_instance_profiles.py,sha256=QUyu30xid60BFaemp-q0y9FgNsHaIQyQnQ8gHUzWC4k,2211
 cartography/intel/aws/identitycenter.py,sha256=C2EOfwQO1zBjePAXtreUcJIy7RU__ior3liRT4SpGO8,9544
-cartography/intel/aws/inspector.py,sha256=rD_O3DUGv5-GSMNqDzb11Ps5bX1sIo_JDq3UTAGvUpE,11168
+cartography/intel/aws/inspector.py,sha256=G8a60uuFCuENvR2DNJ9VmWEJwG9UDENaC1buULvuSZc,12415
 cartography/intel/aws/kms.py,sha256=RtCxNG-Den-f4Il-NO3YL_-BFUmg1qFt4lNucue9SQM,12130
 cartography/intel/aws/lambda_function.py,sha256=cutCsMnvyJB8vKUP0fHORrhijw944cXSoLKOHYdi6SM,10389
 cartography/intel/aws/organizations.py,sha256=m5qk60N6pe7iKLN-Rtfg9aYv7OozoloSkcsuePd-RGs,4680
@@ -427,8 +427,8 @@ cartography/models/aws/identitycenter/awsidentitycenter.py,sha256=-ybcEgdZjwF0RI
 cartography/models/aws/identitycenter/awspermissionset.py,sha256=12kDKzsJQOHg6d9dlX98sM9gKXGMQh7FaxzlJov7Dgo,3651
 cartography/models/aws/identitycenter/awsssouser.py,sha256=EWnyX5ev2aaK64YjKGuJSoknBF5b_kS5xc0kDf_0xfo,2886
 cartography/models/aws/inspector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cartography/models/aws/inspector/findings.py,sha256=z6fVygv8rmlYB1huX64jDgrCEjoTdLkoeKIVeoMk9wI,6469
-cartography/models/aws/inspector/packages.py,sha256=4TWu9EiS8aOR-n5z0FqG7564t-GKS4zrD4THX_2NKRo,3348
+cartography/models/aws/inspector/findings.py,sha256=ikm0vvjhGK_-BDt5VGyS4_0KwJsq81T6VgROq9HK2NE,8188
+cartography/models/aws/inspector/packages.py,sha256=IoJT7w6n8Vx3vGGJPWKDgJLRpioqtOS5iMYZLnX7thg,2113
 cartography/models/aws/s3/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/models/aws/s3/account_public_access_block.py,sha256=L6AS0OncfSOWHP4pOXshnJFDPwnWqywzUIeUppJcw-Q,2256
 cartography/models/aws/s3/notification.py,sha256=SF1VvCP_2WVh8K29d4ms8MUcg9AyO_MN8vCgzLFlGAs,1017
@@ -542,9 +542,9 @@ cartography/models/trivy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 cartography/models/trivy/findings.py,sha256=SgI_h1aRyR20uAHvuXIZ1T6r4IZJt6SVhxRaF2bTsm0,3085
 cartography/models/trivy/fix.py,sha256=ho9ENVl9HSXqyggyCwR6ilkOBKDxpQ7rGibo_j21NA4,2587
 cartography/models/trivy/package.py,sha256=IwO1RZZ-MFRtNbt8Cq6YFl6fdNJMFmULnJkkK8Q4rL4,2809
-cartography-0.107.0rc1.dist-info/licenses/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
-cartography-0.107.0rc1.dist-info/METADATA,sha256=4iitm8S4B9hie7yMMuR5tGdbOPOvBPqucuCHqXUXVnM,12861
-cartography-0.107.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cartography-0.107.0rc1.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
-cartography-0.107.0rc1.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
-cartography-0.107.0rc1.dist-info/RECORD,,
+cartography-0.107.0rc2.dist-info/licenses/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
+cartography-0.107.0rc2.dist-info/METADATA,sha256=fE9Ro2jtKzFQVmrKpCA8YSn_oUzG-ds1qs0OQ6rGzpQ,12861
+cartography-0.107.0rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cartography-0.107.0rc2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
+cartography-0.107.0rc2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
+cartography-0.107.0rc2.dist-info/RECORD,,