cartography 0.97.1__tar.gz → 0.98.0rc1__tar.gz

{cartography-0.97.1 → cartography-0.98.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.97.1
+Version: 0.98.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography/data/indexes.cypher

@@ -161,12 +161,8 @@ CREATE INDEX IF NOT EXISTS FOR (n:GCPSubnet) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPSubnet) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPVpc) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPVpc) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:GitHubOrganization) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:GitHubOrganization) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GitHubRepository) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GitHubRepository) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:GitHubUser) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:GitHubUser) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GKECluster) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GKECluster) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GSuiteGroup) ON (n.email);

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography/data/jobs/analysis/aws_ec2_asset_exposure.json

@@ -1,12 +1,27 @@
 {
   "statements": [
   {
-    "query": "MATCH (n) where n.exposed_internet IS NOT NULL AND labels(n) IN ['AutoScalingGroup', 'EC2Instance', 'LoadBalancer', 'LoadBalancerV2'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted",
+    "query": "MATCH (n:AutoScalingGroup) where n.exposed_internet IS NOT NULL WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type",
     "iterative": true,
     "iterationsize": 1000
   },
   {
-    "query": "MATCH (:IpRange{id: '0.0.0.0/0'})-[:MEMBER_OF_IP_RULE]->(:IpPermissionInbound)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(group:EC2SecurityGroup)<-[:MEMBER_OF_EC2_SECURITY_GROUP|NETWORK_INTERFACE*..2]-(instance:EC2Instance)\nWITH instance\nWHERE (instance.publicipaddress IS NOT NULL) AND (instance.exposed_internet_type IS NULL) OR (NOT 'direct' IN instance.exposed_internet_type)\nSET instance.exposed_internet = true, instance.exposed_internet_type = coalesce(instance.exposed_internet_type , []) + 'direct';",
+    "query": "MATCH (n:EC2Instance) where n.exposed_internet IS NOT NULL WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type",
+    "iterative": true,
+    "iterationsize": 1000
+  },
+  {
+    "query": "MATCH (n:LoadBalancer) where n.exposed_internet IS NOT NULL WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type",
+    "iterative": true,
+    "iterationsize": 1000
+  },
+  {
+    "query": "MATCH (n:LoadBalancerV2) where n.exposed_internet IS NOT NULL WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type",
+    "iterative": true,
+    "iterationsize": 1000
+  },
+  {
+    "query": "MATCH (:IpRange{id: '0.0.0.0/0'})-[:MEMBER_OF_IP_RULE]->(:IpPermissionInbound)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(group:EC2SecurityGroup)<-[:MEMBER_OF_EC2_SECURITY_GROUP|NETWORK_INTERFACE*..2]-(instance:EC2Instance) WITH instance WHERE (instance.publicipaddress IS NOT NULL) AND (instance.exposed_internet_type IS NULL OR NOT 'direct' IN instance.exposed_internet_type) SET instance.exposed_internet = true, instance.exposed_internet_type = CASE WHEN instance.exposed_internet_type IS NULL THEN ['direct'] WHEN NOT 'direct' IN instance.exposed_internet_type THEN instance.exposed_internet_type + ['direct'] ELSE instance.exposed_internet_type END;",
     "iterative": false
   },
   {

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json

@@ -1,7 +1,7 @@
 {
   "statements": [
   {
-    "query": "MATCH (n) where n.exposed_internet IS NOT NULL AND labels(n) IN ['GCPInstance'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted",
+    "query": "MATCH (n:GCPInstance) where n.exposed_internet IS NOT NULL WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type",
     "iterative": true,
     "iterationsize": 1000,
     "__comment__": "Delete exposed_internet off nodes so we can start fresh"

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography/graph/cleanupbuilder.py

@@ -16,7 +16,17 @@ def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
     """
     Generates queries to clean up stale nodes and relationships from the given CartographyNodeSchema.
     Note that auto-cleanups for a node with no relationships is not currently supported.
+
     Algorithm:
+    1. If node_schema has no relationships at all, return empty.
+
+    Otherwise,
+
+    1. If node_schema doesn't have a sub_resource relationship, generate queries only to clean up its other
+    relationships. No nodes will be cleaned up.
+
+    Otherwise,
+
     1. First delete all stale nodes attached to the node_schema's sub resource
     2. Delete all stale node to sub resource relationships
         - We don't expect this to be very common (never for AWS resources, at least), but in case it is possible for an
@@ -25,11 +35,16 @@ def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
     :param node_schema: The given CartographyNodeSchema
     :return: A list of Neo4j queries to clean up nodes and relationships.
     """
+    if not node_schema.sub_resource_relationship and not node_schema.other_relationships:
+        return []
+
     if not node_schema.sub_resource_relationship:
-        raise ValueError(
-            "Auto-creating a cleanup job for a node_schema without a sub resource relationship is not supported. "
-            f'Please check the class definition of "{node_schema.__class__.__name__}".',
-        )
+        queries = []
+        other_rels = node_schema.other_relationships.rels if node_schema.other_relationships else []
+        for rel in other_rels:
+            query = _build_cleanup_rel_query_no_sub_resource(node_schema, rel)
+            queries.append(query)
+        return queries
 
     result = _build_cleanup_node_and_rel_queries(node_schema, node_schema.sub_resource_relationship)
     if node_schema.other_relationships:
@@ -41,6 +56,34 @@ def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
     return result
 
 
+def _build_cleanup_rel_query_no_sub_resource(
+        node_schema: CartographyNodeSchema,
+        selected_relationship: CartographyRelSchema,
+) -> str:
+    """
+    Helper function to delete stale relationships for node_schemas that have no sub resource relationship defined.
+    """
+    if node_schema.sub_resource_relationship:
+        raise ValueError(
+            f'Expected {node_schema.label} to not exist. '
+            'This function is intended for node_schemas without sub_resource_relationships.',
+        )
+    # Ensure the node is attached to the sub resource and delete the node
+    query_template = Template(
+        """
+        MATCH (n:$node_label)
+        $selected_rel_clause
+        WHERE r.lastupdated <> $UPDATE_TAG
+        WITH r LIMIT $LIMIT_SIZE
+        DELETE r;
+        """,
+    )
+    return query_template.safe_substitute(
+        node_label=node_schema.label,
+        selected_rel_clause=_build_selected_rel_clause(selected_relationship),
+    )
+
+
 def _build_cleanup_node_and_rel_queries(
         node_schema: CartographyNodeSchema,
         selected_relationship: CartographyRelSchema,

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography/intel/aws/ec2/instances.py

@@ -13,7 +13,7 @@ from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
 from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
 from cartography.models.aws.ec2.instances import EC2InstanceSchema
-from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
+from cartography.models.aws.ec2.keypair_instance import EC2KeyPairInstanceSchema
 from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema
 from cartography.models.aws.ec2.reservations import EC2ReservationSchema
 from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema
@@ -193,16 +193,17 @@ def load_ec2_subnets(
 
 
 @timeit
-def load_ec2_key_pairs(
+def load_ec2_keypair_instances(
         neo4j_session: neo4j.Session,
         key_pair_list: List[Dict[str, Any]],
         region: str,
         current_aws_account_id: str,
         update_tag: int,
 ) -> None:
+    # Load EC2 keypairs as known by describe-instances.
     load(
         neo4j_session,
-        EC2KeyPairSchema(),
+        EC2KeyPairInstanceSchema(),
         key_pair_list,
         Region=region,
         AWS_ID=current_aws_account_id,
@@ -299,7 +300,7 @@ def load_ec2_instance_data(
     load_ec2_instance_nodes(neo4j_session, instance_list, region, current_aws_account_id, update_tag)
     load_ec2_subnets(neo4j_session, subnet_list, region, current_aws_account_id, update_tag)
     load_ec2_security_groups(neo4j_session, sg_list, region, current_aws_account_id, update_tag)
-    load_ec2_key_pairs(neo4j_session, key_pair_list, region, current_aws_account_id, update_tag)
+    load_ec2_keypair_instances(neo4j_session, key_pair_list, region, current_aws_account_id, update_tag)
     load_ec2_network_interfaces(neo4j_session, nic_list, region, current_aws_account_id, update_tag)
     load_ec2_instance_ebs_volumes(neo4j_session, ebs_volumes_list, region, current_aws_account_id, update_tag)
 

cartography-0.98.0rc1/cartography/intel/aws/ec2/key_pairs.py

@@ -0,0 +1,80 @@
+import logging
+from typing import Any
+from typing import Dict
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.ec2.keypair import EC2KeyPairSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_ec2_key_pairs(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    return client.describe_key_pairs()['KeyPairs']
+
+
+def transform_ec2_key_pairs(
+        key_pairs: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+) -> list[dict[str, Any]]:
+    transformed_key_pairs = []
+    for key_pair in key_pairs:
+        key_name = key_pair["KeyName"]
+        transformed_key_pairs.append({
+            'KeyPairArn': f'arn:aws:ec2:{region}:{current_aws_account_id}:key-pair/{key_name}',
+            'KeyName': key_name,
+            'KeyFingerprint': key_pair.get("KeyFingerprint"),
+        })
+    return transformed_key_pairs
+
+
+@timeit
+def load_ec2_key_pairs(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    # Load EC2 keypairs as known by describe-key-pairs
+    logger.info(f"Loading {len(data)} EC2 keypairs for region '{region}' into graph.")
+    load(
+        neo4j_session,
+        EC2KeyPairSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup_ec2_key_pairs(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+    GraphJob.from_node_schema(EC2KeyPairSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_ec2_key_pairs(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info("Syncing EC2 key pairs for region '%s' in account '%s'.", region, current_aws_account_id)
+        data = get_ec2_key_pairs(boto3_session, region)
+        transformed_data = transform_ec2_key_pairs(data, region, current_aws_account_id)
+        load_ec2_key_pairs(neo4j_session, transformed_data, region, current_aws_account_id, update_tag)
+    cleanup_ec2_key_pairs(neo4j_session, common_job_parameters)

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography/intel/github/users.py

@@ -8,13 +8,13 @@ from typing import Tuple
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.github.util import fetch_all
 from cartography.models.github.orgs import GitHubOrganizationSchema
 from cartography.models.github.users import GitHubOrganizationUserSchema
 from cartography.models.github.users import GitHubUnaffiliatedUserSchema
 from cartography.stats import get_stats_client
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -210,6 +210,13 @@ def load_organization(
     )
 
 
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.info("Cleaning up GitHub users")
+    GraphJob.from_node_schema(GitHubOrganizationUserSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(GitHubUnaffiliatedUserSchema(), common_job_parameters).run(neo4j_session)
+
+
 @timeit
 def sync(
         neo4j_session: neo4j.Session,
@@ -236,8 +243,8 @@ def sync(
         neo4j_session, GitHubUnaffiliatedUserSchema(), processed_unaffiliated_user_data, org_data,
         common_job_parameters['UPDATE_TAG'],
     )
-    # no automated cleanup job for users because user node has no sub_resource_relationship
-    run_cleanup_job('github_org_and_users_cleanup.json', neo4j_session, common_job_parameters)
+    cleanup(neo4j_session, common_job_parameters)
+
     merge_module_sync_metadata(
         neo4j_session,
         group_type='GitHubOrganization',

cartography-0.97.1/cartography/models/aws/ec2/keypairs.py → cartography-0.98.0rc1/cartography/models/aws/ec2/keypair.py

@@ -3,19 +3,23 @@ from dataclasses import dataclass
 from cartography.models.core.common import PropertyRef
 from cartography.models.core.nodes import CartographyNodeProperties
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
 from cartography.models.core.relationships import make_target_node_matcher
-from cartography.models.core.relationships import OtherRelationships
 from cartography.models.core.relationships import TargetNodeMatcher
 
 
 @dataclass(frozen=True)
 class EC2KeyPairNodeProperties(CartographyNodeProperties):
+    """
+    Properties for EC2 keypairs from describe-key-pairs
+    """
     id: PropertyRef = PropertyRef('KeyPairArn')
     arn: PropertyRef = PropertyRef('KeyPairArn', extra_index=True)
     keyname: PropertyRef = PropertyRef('KeyName')
+    keyfingerprint: PropertyRef = PropertyRef('KeyFingerprint')
     region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
     lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
 
@@ -27,38 +31,24 @@ class EC2KeyPairToAwsAccountRelProperties(CartographyRelProperties):
 
 @dataclass(frozen=True)
 class EC2KeyPairToAWSAccount(CartographyRelSchema):
+    """
+    Relationship schema for EC2 keypairs to AWS Accounts
+    """
     target_node_label: str = 'AWSAccount'
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
         {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
     )
     direction: LinkDirection = LinkDirection.INWARD
-    rel_label: str = "RESOURCE"
+    rel_label: str = 'RESOURCE'
     properties: EC2KeyPairToAwsAccountRelProperties = EC2KeyPairToAwsAccountRelProperties()
 
 
-@dataclass(frozen=True)
-class EC2KeyPairToEC2InstanceRelProperties(CartographyRelProperties):
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
-
-
-@dataclass(frozen=True)
-class EC2KeyPairToEC2Instance(CartographyRelSchema):
-    target_node_label: str = 'EC2Instance'
-    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {'id': PropertyRef('InstanceId')},
-    )
-    direction: LinkDirection = LinkDirection.OUTWARD
-    rel_label: str = "SSH_LOGIN_TO"
-    properties: EC2KeyPairToEC2InstanceRelProperties = EC2KeyPairToEC2InstanceRelProperties()
-
-
 @dataclass(frozen=True)
 class EC2KeyPairSchema(CartographyNodeSchema):
+    """
+    Schema for EC2 keypairs from describe-key-pairs
+    """
     label: str = 'EC2KeyPair'
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(['KeyPair'])
     properties: EC2KeyPairNodeProperties = EC2KeyPairNodeProperties()
     sub_resource_relationship: EC2KeyPairToAWSAccount = EC2KeyPairToAWSAccount()
-    other_relationships: OtherRelationships = OtherRelationships(
-        [
-            EC2KeyPairToEC2Instance(),
-        ],
-    )

cartography-0.98.0rc1/cartography/models/aws/ec2/keypair_instance.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class EC2KeyPairInstanceNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('KeyPairArn')
+    arn: PropertyRef = PropertyRef('KeyPairArn', extra_index=True)
+    keyname: PropertyRef = PropertyRef('KeyName')
+    region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2KeyPairInstanceToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2KeyPairInstanceToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: EC2KeyPairInstanceToAwsAccountRelProperties = EC2KeyPairInstanceToAwsAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2KeyPairInstanceToEC2InstanceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2KeyPairInstanceToEC2Instance(CartographyRelSchema):
+    target_node_label: str = 'EC2Instance'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('InstanceId')},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "SSH_LOGIN_TO"
+    properties: EC2KeyPairInstanceToEC2InstanceRelProperties = EC2KeyPairInstanceToEC2InstanceRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2KeyPairInstanceSchema(CartographyNodeSchema):
+    """
+    EC2 keypairs as known by describe-instances.
+    """
+    label: str = 'EC2KeyPair'
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(['KeyPair'])
+    properties: EC2KeyPairInstanceNodeProperties = EC2KeyPairInstanceNodeProperties()
+    sub_resource_relationship: EC2KeyPairInstanceToAWSAccount = EC2KeyPairInstanceToAWSAccount()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            EC2KeyPairInstanceToEC2Instance(),
+        ],
+    )

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.97.1
+Version: 0.98.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.97.1 → cartography-0.98.0rc1}/cartography.egg-info/SOURCES.txt

@@ -112,7 +112,6 @@ cartography/data/jobs/cleanup/gcp_crm_project_cleanup.json
 cartography/data/jobs/cleanup/gcp_dns_cleanup.json
 cartography/data/jobs/cleanup/gcp_gke_cluster_cleanup.json
 cartography/data/jobs/cleanup/gcp_storage_bucket_cleanup.json
-cartography/data/jobs/cleanup/github_org_and_users_cleanup.json
 cartography/data/jobs/cleanup/github_repos_cleanup.json
 cartography/data/jobs/cleanup/gsuite_ingest_groups_cleanup.json
 cartography/data/jobs/cleanup/gsuite_ingest_users_cleanup.json
@@ -298,7 +297,8 @@ cartography/models/aws/ec2/__init__.py
 cartography/models/aws/ec2/auto_scaling_groups.py
 cartography/models/aws/ec2/images.py
 cartography/models/aws/ec2/instances.py
-cartography/models/aws/ec2/keypairs.py
+cartography/models/aws/ec2/keypair.py
+cartography/models/aws/ec2/keypair_instance.py
 cartography/models/aws/ec2/launch_configurations.py
 cartography/models/aws/ec2/launch_template_versions.py
 cartography/models/aws/ec2/launch_templates.py

{cartography-0.97.1 → cartography-0.98.0rc1}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = '0.97.1'
+__version__ = '0.98.0rc1'
 
 
 setup(

cartography-0.97.1/cartography/data/jobs/cleanup/github_org_and_users_cleanup.json

@@ -1,28 +0,0 @@
-{
-  "statements": [{
-    "query": "MATCH (n:GitHubUser) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (n:GitHubOrganization) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:OWNER]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:MEMBER_OF]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:UNAFFILIATED]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  }],
-  "name": "cleanup GitHub users data"
-}

cartography-0.97.1/cartography/intel/aws/ec2/key_pairs.py

@@ -1,71 +0,0 @@
-import logging
-from typing import Dict
-from typing import List
-
-import boto3
-import neo4j
-
-from .util import get_botocore_config
-from cartography.graph.job import GraphJob
-from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
-from cartography.util import aws_handle_regions
-from cartography.util import timeit
-
-logger = logging.getLogger(__name__)
-
-
-@timeit
-@aws_handle_regions
-def get_ec2_key_pairs(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
-    return client.describe_key_pairs()['KeyPairs']
-
-
-@timeit
-def load_ec2_key_pairs(
-    neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str,
-    update_tag: int,
-) -> None:
-    ingest_key_pair = """
-    MERGE (keypair:KeyPair:EC2KeyPair{arn: $ARN, id: $ARN})
-    ON CREATE SET keypair.firstseen = timestamp()
-    SET keypair.keyname = $KeyName, keypair.keyfingerprint = $KeyFingerprint, keypair.region = $Region,
-    keypair.lastupdated = $update_tag
-    WITH keypair
-    MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (aa)-[r:RESOURCE]->(keypair)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
-    """
-
-    for key_pair in data:
-        key_name = key_pair["KeyName"]
-        key_fingerprint = key_pair.get("KeyFingerprint")
-        key_pair_arn = f'arn:aws:ec2:{region}:{current_aws_account_id}:key-pair/{key_name}'
-
-        neo4j_session.run(
-            ingest_key_pair,
-            ARN=key_pair_arn,
-            KeyName=key_name,
-            KeyFingerprint=key_fingerprint,
-            AWS_ACCOUNT_ID=current_aws_account_id,
-            Region=region,
-            update_tag=update_tag,
-        )
-
-
-@timeit
-def cleanup_ec2_key_pairs(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    GraphJob.from_node_schema(EC2KeyPairSchema(), common_job_parameters).run(neo4j_session)
-
-
-@timeit
-def sync_ec2_key_pairs(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
-) -> None:
-    for region in regions:
-        logger.info("Syncing EC2 key pairs for region '%s' in account '%s'.", region, current_aws_account_id)
-        data = get_ec2_key_pairs(boto3_session, region)
-        load_ec2_key_pairs(neo4j_session, data, region, current_aws_account_id, update_tag)
-    cleanup_ec2_key_pairs(neo4j_session, common_job_parameters)