cartography 0.85.0__tar.gz → 0.85.1__tar.gz

{cartography-0.85.0/cartography.egg-info → cartography-0.85.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.85.0
+Version: 0.85.1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.85.0 → cartography-0.85.1}/cartography/data/indexes.cypher

@@ -119,8 +119,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:EKSCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:EKSCluster) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.lastupdated);

{cartography-0.85.0 → cartography-0.85.1}/cartography/data/jobs/analysis/aws_s3acl_analysis.json

@@ -1,22 +1,27 @@
 {
   "statements": [
     {
+      "__comment__": "READ -> ListBucket, ListBucketVersions, ListBucketMultipartUploads",
       "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'READ'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:ListBucket', 's3:ListBucketVersions', 's3:ListBucketMultipartUploads']",
       "iterative": false
     },
     {
-      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'WRITE'\nAND (acl.ownerid = acl.granteeid)\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:DeleteObjectVersion']",
+      "__comment__": "WRITE -> PutObject",
+      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'WRITE'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:PutObject']",
       "iterative": false
     },
     {
-      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'READ_ACP'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:DeleteObjectVersion']",
+      "__comment__": "READ_ACP -> GetBucketAcl",
+      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'READ_ACP'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:GetBucketAcl']",
       "iterative": false
     },
     {
+      "__comment__": "WRITE_ACP -> PutBucketAcl",
       "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'WRITE_ACP'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:PutBucketAcl']",
       "iterative": false
     },
     {
+      "__comment__": "FULL_CONTROL -> Pretty much everything",
       "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'FULL_CONTROL'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:ListBucket', 's3:ListBucketVersions', 's3:ListBucketMultipartUploads', 's3:PutObject', 's3:DeleteObject', 's3:DeleteObjectVersion', 's3:PutBucketAcl']",
       "iterative": false
     }],

{cartography-0.85.0 → cartography-0.85.1}/cartography/intel/aws/ec2/network_interfaces.py

@@ -54,7 +54,7 @@ def transform_network_interface_data(data_list: List[Dict[str, Any]], region: st
         elb_v2_id = None
         elb_match = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', ''))
         if elb_match:
-            elb_v1_id = f'{elb_match[1]}-{elb_match[2]}.elb.{region}.amazonaws.com',
+            elb_v1_id = f'{elb_match[1]}-{elb_match[2]}.elb.{region}.amazonaws.com'
         else:
             elb_match = re.match(r'^ELB (.*)', network_interface.get('Description', ''))
             if elb_match:

cartography-0.85.1/cartography/intel/aws/eks.py

@@ -0,0 +1,106 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.eks.clusters import EKSClusterSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[str]:
+    client = boto3_session.client('eks', region_name=region)
+    clusters: List[str] = []
+    paginator = client.get_paginator('list_clusters')
+    for page in paginator.paginate():
+        clusters.extend(page['clusters'])
+    return clusters
+
+
+@timeit
+def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_name: str) -> Dict:
+    client = boto3_session.client('eks', region_name=region)
+    response = client.describe_cluster(name=cluster_name)
+    return response['cluster']
+
+
+@timeit
+def load_eks_clusters(
+        neo4j_session: neo4j.Session,
+        cluster_data: List[Dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        EKSClusterSchema(),
+        cluster_data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=aws_update_tag,
+    )
+
+
+def _process_logging(cluster: Dict) -> bool:
+    """
+    Parse cluster.logging.clusterLogging to verify if
+    at least one entry has audit logging set to Enabled.
+    """
+    logging: bool = False
+    cluster_logging: Any = cluster.get('logging', {}).get('clusterLogging')
+    if cluster_logging:
+        logging = any(filter(lambda x: 'audit' in x['types'] and x['enabled'], cluster_logging))  # type: ignore
+    return logging
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    logger.info("Running EKS cluster cleanup")
+    GraphJob.from_node_schema(EKSClusterSchema(), common_job_parameters).run(neo4j_session)
+
+
+def transform(cluster_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    transformed_list = []
+    for cluster_name, cluster_dict in cluster_data.items():
+        transformed_dict = cluster_dict.copy()
+        transformed_dict['ClusterLogging'] = _process_logging(transformed_dict)
+        transformed_dict['ClusterEndpointPublic'] = transformed_dict.get('resourcesVpcConfig', {}).get(
+            'endpointPublicAccess',
+        )
+        if 'createdAt' in transformed_dict:
+            transformed_dict['created_at'] = str(transformed_dict['createdAt'])
+        transformed_list.append(transformed_dict)
+    return transformed_list
+
+
+@timeit
+def sync(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: List[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info("Syncing EKS for region '%s' in account '%s'.", region, current_aws_account_id)
+
+        clusters: List[str] = get_eks_clusters(boto3_session, region)
+        cluster_data = {}
+        for cluster_name in clusters:
+            cluster_data[cluster_name] = get_eks_describe_cluster(boto3_session, region, cluster_name)
+        transformed_list = transform(cluster_data)
+
+        load_eks_clusters(neo4j_session, transformed_list, region, current_aws_account_id, update_tag)
+
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.85.0 → cartography-0.85.1}/cartography/intel/aws/s3.py

@@ -222,7 +222,12 @@ def _is_common_exception(e: Exception, bucket: Dict) -> bool:
 
 
 @timeit
-def _load_s3_acls(neo4j_session: neo4j.Session, acls: Dict, aws_account_id: str, update_tag: int) -> None:
+def _load_s3_acls(
+        neo4j_session: neo4j.Session,
+        acls: List[Dict[str, Any]],
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
     """
     Ingest S3 ACL into neo4j.
     """

cartography-0.85.1/cartography/models/aws/eks/clusters.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class EKSClusterNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('arn')
+    arn: PropertyRef = PropertyRef('arn', extra_index=True)
+    name: PropertyRef = PropertyRef('name', extra_index=True)
+    region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
+    created_at: PropertyRef = PropertyRef('created_at')
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    endpoint: PropertyRef = PropertyRef('endpoint')
+    endpoint_public_access: PropertyRef = PropertyRef('ClusterEndpointPublic')
+    rolearn: PropertyRef = PropertyRef('roleArn')
+    version: PropertyRef = PropertyRef('version')
+    platform_version: PropertyRef = PropertyRef('platformVersion')
+    status: PropertyRef = PropertyRef('status')
+    audit_logging: PropertyRef = PropertyRef('ClusterLogging')
+
+
+@dataclass(frozen=True)
+class EKSClusterToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EKSClusterToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: EKSClusterToAwsAccountRelProperties = EKSClusterToAwsAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class EKSClusterSchema(CartographyNodeSchema):
+    label: str = 'EKSCluster'
+    properties: EKSClusterNodeProperties = EKSClusterNodeProperties()
+    sub_resource_relationship: EKSClusterToAWSAccount = EKSClusterToAWSAccount()

{cartography-0.85.0 → cartography-0.85.1/cartography.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.85.0
+Version: 0.85.1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.85.0 → cartography-0.85.1}/cartography.egg-info/SOURCES.txt

@@ -51,7 +51,6 @@ cartography/data/jobs/cleanup/aws_import_ec2_launch_templates_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ec2_security_groupinfo_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ecr_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ecs_cleanup.json
-cartography/data/jobs/cleanup/aws_import_eks_cleanup.json
 cartography/data/jobs/cleanup/aws_import_elastic_ip_addresses_cleanup.json
 cartography/data/jobs/cleanup/aws_import_elasticache_cleanup.json
 cartography/data/jobs/cleanup/aws_import_es_cleanup.json
@@ -301,6 +300,8 @@ cartography/models/aws/ec2/securitygroup_networkinterface.py
 cartography/models/aws/ec2/subnet_instance.py
 cartography/models/aws/ec2/subnet_networkinterface.py
 cartography/models/aws/ec2/volumes.py
+cartography/models/aws/eks/__init__.py
+cartography/models/aws/eks/clusters.py
 cartography/models/aws/inspector/__init__.py
 cartography/models/aws/inspector/findings.py
 cartography/models/aws/inspector/packages.py

{cartography-0.85.0 → cartography-0.85.1}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = '0.85.0'
+__version__ = '0.85.1'
 
 
 setup(

cartography-0.85.0/cartography/data/jobs/cleanup/aws_import_eks_cleanup.json

@@ -1,15 +0,0 @@
-{
-    "statements": [
-        {
-            "query": "MATCH (n:EKSCluster)<-[:RESOURCE]-(:AWSAccount{id: $AWS_ID}) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-            "iterative": true,
-            "iterationsize": 100
-        },
-        {
-            "query": "MATCH (:EKSCluster)<-[r:RESOURCE]-(:AWSAccount{id: $AWS_ID}) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-            "iterative": true,
-            "iterationsize": 100
-        }
-    ],
-    "name": "cleanup EKSCluster"
-}

cartography-0.85.0/cartography/intel/aws/eks.py

@@ -1,114 +0,0 @@
-import logging
-from typing import Any
-from typing import Dict
-from typing import List
-
-import boto3
-import neo4j
-
-from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
-from cartography.util import timeit
-
-logger = logging.getLogger(__name__)
-
-
-@timeit
-@aws_handle_regions
-def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('eks', region_name=region)
-    clusters: List[Dict] = []
-    paginator = client.get_paginator('list_clusters')
-    for page in paginator.paginate():
-        clusters.extend(page['clusters'])
-    return clusters
-
-
-@timeit
-def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_name: str) -> Dict:
-    client = boto3_session.client('eks', region_name=region)
-    response = client.describe_cluster(name=cluster_name)
-    return response['cluster']
-
-
-@timeit
-def load_eks_clusters(
-    neo4j_session: neo4j.Session, cluster_data: Dict, region: str, current_aws_account_id: str,
-    aws_update_tag: int,
-) -> None:
-    query: str = """
-    MERGE (cluster:EKSCluster{id: $ClusterArn})
-    ON CREATE SET cluster.firstseen = timestamp(),
-                cluster.arn = $ClusterArn,
-                cluster.name = $ClusterName,
-                cluster.region = $Region,
-                cluster.created_at = $CreatedAt
-    SET cluster.lastupdated = $aws_update_tag,
-        cluster.endpoint = $ClusterEndpoint,
-        cluster.endpoint_public_access = $ClusterEndointPublic,
-        cluster.rolearn = $ClusterRoleArn,
-        cluster.version = $ClusterVersion,
-        cluster.platform_version = $ClusterPlatformVersion,
-        cluster.status = $ClusterStatus,
-        cluster.audit_logging = $ClusterLogging
-    WITH cluster
-    MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (owner)-[r:RESOURCE]->(cluster)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag
-    """
-
-    for cd in cluster_data:
-        cluster = cluster_data[cd]
-        neo4j_session.run(
-            query,
-            ClusterArn=cluster['arn'],
-            ClusterName=cluster['name'],
-            ClusterEndpoint=cluster.get('endpoint'),
-            ClusterEndointPublic=cluster.get('resourcesVpcConfig', {}).get('endpointPublicAccess'),
-            ClusterRoleArn=cluster.get('roleArn'),
-            ClusterVersion=cluster.get('version'),
-            ClusterPlatformVersion=cluster.get('platformVersion'),
-            ClusterStatus=cluster.get('status'),
-            CreatedAt=str(cluster.get('createdAt')),
-            ClusterLogging=_process_logging(cluster),
-            Region=region,
-            aws_update_tag=aws_update_tag,
-            AWS_ACCOUNT_ID=current_aws_account_id,
-        )
-
-
-def _process_logging(cluster: Dict) -> bool:
-    """
-    Parse cluster.logging.clusterLogging to verify if
-    at least one entry has audit logging set to Enabled.
-    """
-    logging: bool = False
-    cluster_logging: Any = cluster.get('logging', {}).get('clusterLogging')
-    if cluster_logging:
-        logging = any(filter(lambda x: 'audit' in x['types'] and x['enabled'], cluster_logging))  # type: ignore
-    return logging
-
-
-@timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('aws_import_eks_cleanup.json', neo4j_session, common_job_parameters)
-
-
-@timeit
-def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
-) -> None:
-    for region in regions:
-        logger.info("Syncing EKS for region '%s' in account '%s'.", region, current_aws_account_id)
-
-        clusters: List[Dict] = get_eks_clusters(boto3_session, region)
-
-        cluster_data: Dict = {}
-        for cluster_name in clusters:
-            cluster_data[cluster_name] = get_eks_describe_cluster(boto3_session, region, cluster_name)  # type: ignore
-
-        load_eks_clusters(neo4j_session, cluster_data, region, current_aws_account_id, update_tag)
-
-    cleanup(neo4j_session, common_job_parameters)