cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/__main__.py +1 -2
- cartography/_version.py +2 -2
- cartography/cli.py +302 -253
- cartography/client/core/tx.py +39 -18
- cartography/config.py +4 -0
- cartography/driftdetect/__main__.py +1 -2
- cartography/driftdetect/add_shortcut.py +10 -2
- cartography/driftdetect/cli.py +71 -75
- cartography/driftdetect/detect_deviations.py +7 -3
- cartography/driftdetect/get_states.py +20 -8
- cartography/driftdetect/model.py +5 -5
- cartography/driftdetect/serializers.py +8 -6
- cartography/driftdetect/storage.py +2 -2
- cartography/graph/cleanupbuilder.py +35 -15
- cartography/graph/job.py +46 -17
- cartography/graph/querybuilder.py +165 -80
- cartography/graph/statement.py +35 -26
- cartography/intel/analysis.py +4 -1
- cartography/intel/aws/__init__.py +114 -55
- cartography/intel/aws/apigateway.py +134 -63
- cartography/intel/aws/cloudtrail.py +127 -0
- cartography/intel/aws/config.py +56 -20
- cartography/intel/aws/dynamodb.py +108 -40
- cartography/intel/aws/ec2/__init__.py +2 -2
- cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
- cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
- cartography/intel/aws/ec2/images.py +49 -20
- cartography/intel/aws/ec2/instances.py +234 -136
- cartography/intel/aws/ec2/internet_gateways.py +40 -11
- cartography/intel/aws/ec2/key_pairs.py +44 -20
- cartography/intel/aws/ec2/launch_templates.py +101 -59
- cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
- cartography/intel/aws/ec2/load_balancers.py +82 -42
- cartography/intel/aws/ec2/network_acls.py +89 -65
- cartography/intel/aws/ec2/network_interfaces.py +146 -87
- cartography/intel/aws/ec2/reserved_instances.py +45 -16
- cartography/intel/aws/ec2/route_tables.py +138 -98
- cartography/intel/aws/ec2/security_groups.py +71 -21
- cartography/intel/aws/ec2/snapshots.py +61 -22
- cartography/intel/aws/ec2/subnets.py +54 -18
- cartography/intel/aws/ec2/tgw.py +100 -34
- cartography/intel/aws/ec2/util.py +1 -1
- cartography/intel/aws/ec2/volumes.py +69 -41
- cartography/intel/aws/ec2/vpc.py +37 -12
- cartography/intel/aws/ec2/vpc_peerings.py +83 -24
- cartography/intel/aws/ecr.py +88 -32
- cartography/intel/aws/ecs.py +83 -47
- cartography/intel/aws/eks.py +55 -29
- cartography/intel/aws/elasticache.py +42 -18
- cartography/intel/aws/elasticsearch.py +57 -20
- cartography/intel/aws/emr.py +61 -23
- cartography/intel/aws/iam.py +401 -145
- cartography/intel/aws/iam_instance_profiles.py +22 -22
- cartography/intel/aws/identitycenter.py +71 -37
- cartography/intel/aws/inspector.py +159 -89
- cartography/intel/aws/kms.py +92 -38
- cartography/intel/aws/lambda_function.py +103 -34
- cartography/intel/aws/organizations.py +30 -10
- cartography/intel/aws/permission_relationships.py +133 -51
- cartography/intel/aws/rds.py +249 -85
- cartography/intel/aws/redshift.py +107 -46
- cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
- cartography/intel/aws/resources.py +53 -46
- cartography/intel/aws/route53.py +108 -61
- cartography/intel/aws/s3.py +168 -83
- cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
- cartography/intel/aws/secretsmanager.py +24 -12
- cartography/intel/aws/securityhub.py +20 -9
- cartography/intel/aws/sns.py +166 -0
- cartography/intel/aws/sqs.py +60 -28
- cartography/intel/aws/ssm.py +70 -30
- cartography/intel/aws/util/arns.py +7 -7
- cartography/intel/aws/util/common.py +31 -4
- cartography/intel/azure/__init__.py +78 -19
- cartography/intel/azure/compute.py +101 -27
- cartography/intel/azure/cosmosdb.py +496 -170
- cartography/intel/azure/sql.py +296 -105
- cartography/intel/azure/storage.py +322 -113
- cartography/intel/azure/subscription.py +39 -23
- cartography/intel/azure/tenant.py +13 -4
- cartography/intel/azure/util/credentials.py +95 -55
- cartography/intel/bigfix/__init__.py +2 -2
- cartography/intel/bigfix/computers.py +93 -65
- cartography/intel/create_indexes.py +3 -2
- cartography/intel/crowdstrike/__init__.py +11 -9
- cartography/intel/crowdstrike/endpoints.py +5 -1
- cartography/intel/crowdstrike/spotlight.py +8 -3
- cartography/intel/cve/__init__.py +46 -13
- cartography/intel/cve/feed.py +48 -12
- cartography/intel/digitalocean/__init__.py +22 -13
- cartography/intel/digitalocean/compute.py +75 -108
- cartography/intel/digitalocean/management.py +44 -80
- cartography/intel/digitalocean/platform.py +48 -43
- cartography/intel/dns.py +36 -10
- cartography/intel/duo/__init__.py +21 -16
- cartography/intel/duo/api_host.py +14 -9
- cartography/intel/duo/endpoints.py +50 -45
- cartography/intel/duo/groups.py +18 -14
- cartography/intel/duo/phones.py +37 -34
- cartography/intel/duo/tokens.py +26 -23
- cartography/intel/duo/users.py +54 -50
- cartography/intel/duo/web_authn_credentials.py +30 -25
- cartography/intel/entra/__init__.py +25 -7
- cartography/intel/entra/ou.py +112 -0
- cartography/intel/entra/users.py +69 -63
- cartography/intel/gcp/__init__.py +185 -49
- cartography/intel/gcp/compute.py +418 -231
- cartography/intel/gcp/crm.py +96 -43
- cartography/intel/gcp/dns.py +60 -19
- cartography/intel/gcp/gke.py +72 -38
- cartography/intel/gcp/iam.py +61 -41
- cartography/intel/gcp/storage.py +84 -55
- cartography/intel/github/__init__.py +13 -11
- cartography/intel/github/repos.py +270 -137
- cartography/intel/github/teams.py +170 -88
- cartography/intel/github/users.py +70 -39
- cartography/intel/github/util.py +36 -34
- cartography/intel/gsuite/__init__.py +47 -26
- cartography/intel/gsuite/api.py +73 -30
- cartography/intel/jamf/__init__.py +19 -1
- cartography/intel/jamf/computers.py +30 -7
- cartography/intel/jamf/util.py +7 -2
- cartography/intel/kandji/__init__.py +6 -3
- cartography/intel/kandji/devices.py +14 -8
- cartography/intel/kubernetes/namespaces.py +7 -4
- cartography/intel/kubernetes/pods.py +7 -4
- cartography/intel/kubernetes/services.py +8 -4
- cartography/intel/lastpass/__init__.py +2 -2
- cartography/intel/lastpass/users.py +23 -12
- cartography/intel/oci/__init__.py +44 -11
- cartography/intel/oci/iam.py +134 -38
- cartography/intel/oci/organizations.py +13 -6
- cartography/intel/oci/utils.py +43 -20
- cartography/intel/okta/__init__.py +66 -15
- cartography/intel/okta/applications.py +42 -20
- cartography/intel/okta/awssaml.py +93 -33
- cartography/intel/okta/factors.py +16 -4
- cartography/intel/okta/groups.py +56 -29
- cartography/intel/okta/organization.py +5 -1
- cartography/intel/okta/origins.py +6 -2
- cartography/intel/okta/roles.py +15 -5
- cartography/intel/okta/users.py +20 -8
- cartography/intel/okta/utils.py +6 -4
- cartography/intel/pagerduty/__init__.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +18 -6
- cartography/intel/pagerduty/schedules.py +12 -4
- cartography/intel/pagerduty/services.py +11 -4
- cartography/intel/pagerduty/teams.py +8 -3
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/semgrep/__init__.py +24 -6
- cartography/intel/semgrep/dependencies.py +50 -28
- cartography/intel/semgrep/deployment.py +3 -1
- cartography/intel/semgrep/findings.py +42 -18
- cartography/intel/snipeit/__init__.py +17 -3
- cartography/intel/snipeit/asset.py +12 -6
- cartography/intel/snipeit/user.py +8 -5
- cartography/intel/snipeit/util.py +9 -4
- cartography/models/aws/apigateway.py +21 -17
- cartography/models/aws/apigatewaycertificate.py +28 -22
- cartography/models/aws/apigatewayresource.py +28 -20
- cartography/models/aws/apigatewaystage.py +33 -25
- cartography/models/aws/cloudtrail/__init__.py +0 -0
- cartography/models/aws/cloudtrail/trail.py +61 -0
- cartography/models/aws/dynamodb/gsi.py +30 -22
- cartography/models/aws/dynamodb/tables.py +25 -17
- cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
- cartography/models/aws/ec2/images.py +36 -34
- cartography/models/aws/ec2/instances.py +51 -45
- cartography/models/aws/ec2/keypair.py +21 -16
- cartography/models/aws/ec2/keypair_instance.py +28 -21
- cartography/models/aws/ec2/launch_configurations.py +30 -26
- cartography/models/aws/ec2/launch_template_versions.py +48 -38
- cartography/models/aws/ec2/launch_templates.py +21 -17
- cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
- cartography/models/aws/ec2/load_balancers.py +47 -37
- cartography/models/aws/ec2/network_acl_rules.py +38 -30
- cartography/models/aws/ec2/network_acls.py +38 -29
- cartography/models/aws/ec2/networkinterface_instance.py +52 -39
- cartography/models/aws/ec2/networkinterfaces.py +53 -37
- cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
- cartography/models/aws/ec2/reservations.py +18 -14
- cartography/models/aws/ec2/route_table_associations.py +44 -34
- cartography/models/aws/ec2/route_tables.py +50 -43
- cartography/models/aws/ec2/routes.py +45 -37
- cartography/models/aws/ec2/securitygroup_instance.py +29 -20
- cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
- cartography/models/aws/ec2/subnet_instance.py +24 -19
- cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
- cartography/models/aws/ec2/volumes.py +47 -40
- cartography/models/aws/eks/clusters.py +23 -21
- cartography/models/aws/emr.py +32 -30
- cartography/models/aws/iam/instanceprofile.py +33 -24
- cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
- cartography/models/aws/identitycenter/awspermissionset.py +37 -29
- cartography/models/aws/identitycenter/awsssouser.py +23 -21
- cartography/models/aws/inspector/findings.py +77 -65
- cartography/models/aws/inspector/packages.py +35 -29
- cartography/models/aws/s3/__init__.py +0 -0
- cartography/models/aws/s3/account_public_access_block.py +51 -0
- cartography/models/aws/sns/__init__.py +0 -0
- cartography/models/aws/sns/topic.py +50 -0
- cartography/models/aws/ssm/instance_information.py +51 -39
- cartography/models/aws/ssm/instance_patch.py +32 -26
- cartography/models/bigfix/bigfix_computer.py +42 -38
- cartography/models/bigfix/bigfix_root.py +3 -3
- cartography/models/core/common.py +12 -10
- cartography/models/core/nodes.py +5 -2
- cartography/models/core/relationships.py +14 -6
- cartography/models/crowdstrike/hosts.py +37 -35
- cartography/models/cve/cve.py +34 -32
- cartography/models/cve/cve_feed.py +6 -6
- cartography/models/digitalocean/__init__.py +0 -0
- cartography/models/digitalocean/account.py +21 -0
- cartography/models/digitalocean/droplet.py +56 -0
- cartography/models/digitalocean/project.py +48 -0
- cartography/models/duo/api_host.py +3 -3
- cartography/models/duo/endpoint.py +43 -41
- cartography/models/duo/group.py +14 -14
- cartography/models/duo/phone.py +27 -27
- cartography/models/duo/token.py +16 -16
- cartography/models/duo/user.py +46 -44
- cartography/models/duo/web_authn_credential.py +27 -19
- cartography/models/entra/ou.py +48 -0
- cartography/models/entra/tenant.py +24 -18
- cartography/models/entra/user.py +64 -48
- cartography/models/gcp/iam.py +23 -23
- cartography/models/github/orgs.py +5 -4
- cartography/models/github/teams.py +37 -31
- cartography/models/github/users.py +34 -23
- cartography/models/kandji/device.py +22 -16
- cartography/models/kandji/tenant.py +6 -4
- cartography/models/lastpass/tenant.py +3 -3
- cartography/models/lastpass/user.py +32 -28
- cartography/models/semgrep/dependencies.py +36 -24
- cartography/models/semgrep/deployment.py +5 -5
- cartography/models/semgrep/findings.py +58 -42
- cartography/models/semgrep/locations.py +27 -21
- cartography/models/snipeit/asset.py +30 -21
- cartography/models/snipeit/tenant.py +6 -4
- cartography/models/snipeit/user.py +19 -12
- cartography/stats.py +3 -3
- cartography/sync.py +107 -31
- cartography/util.py +84 -62
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
- cartography-0.103.0rc1.dist-info/RECORD +396 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
- cartography-0.102.0rc2.dist-info/RECORD +0 -381
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -22,12 +22,19 @@ stat_handler = get_stats_client(__name__)
|
|
|
22
22
|
|
|
23
23
|
@timeit
|
|
24
24
|
@aws_handle_regions
|
|
25
|
-
def get_dynamodb_tables(
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
def get_dynamodb_tables(
|
|
26
|
+
boto3_session: boto3.session.Session,
|
|
27
|
+
region: str,
|
|
28
|
+
) -> List[Dict]:
|
|
29
|
+
client = boto3_session.client(
|
|
30
|
+
"dynamodb",
|
|
31
|
+
region_name=region,
|
|
32
|
+
config=get_botocore_config(),
|
|
33
|
+
)
|
|
34
|
+
paginator = client.get_paginator("list_tables")
|
|
28
35
|
dynamodb_tables = []
|
|
29
36
|
for page in paginator.paginate():
|
|
30
|
-
for table_name in page[
|
|
37
|
+
for table_name in page["TableNames"]:
|
|
31
38
|
dynamodb_tables.append(client.describe_table(TableName=table_name))
|
|
32
39
|
return dynamodb_tables
|
|
33
40
|
|
|
@@ -38,33 +45,50 @@ def transform_dynamodb_tables(dynamodb_tables: List, region: str) -> Any:
|
|
|
38
45
|
ddb_gsi_data: List[Dict[str, Any]] = []
|
|
39
46
|
|
|
40
47
|
for table in dynamodb_tables:
|
|
41
|
-
ddb_table_data.append(
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
48
|
+
ddb_table_data.append(
|
|
49
|
+
{
|
|
50
|
+
"Arn": table["Table"]["TableArn"],
|
|
51
|
+
"TableName": table["Table"]["TableName"],
|
|
52
|
+
"Region": region,
|
|
53
|
+
"Rows": table["Table"]["ItemCount"],
|
|
54
|
+
"Size": table["Table"]["TableSizeBytes"],
|
|
55
|
+
"ProvisionedThroughputReadCapacityUnits": table["Table"][
|
|
56
|
+
"ProvisionedThroughput"
|
|
57
|
+
]["ReadCapacityUnits"],
|
|
58
|
+
"ProvisionedThroughputWriteCapacityUnits": table["Table"][
|
|
59
|
+
"ProvisionedThroughput"
|
|
60
|
+
]["WriteCapacityUnits"],
|
|
61
|
+
},
|
|
62
|
+
)
|
|
63
|
+
for gsi in table["Table"].get("GlobalSecondaryIndexes", []):
|
|
64
|
+
ddb_gsi_data.append(
|
|
65
|
+
{
|
|
66
|
+
"Arn": gsi["IndexArn"],
|
|
67
|
+
"TableArn": table["Table"]["TableArn"],
|
|
68
|
+
"Region": region,
|
|
69
|
+
"ProvisionedThroughputReadCapacityUnits": gsi[
|
|
70
|
+
"ProvisionedThroughput"
|
|
71
|
+
]["ReadCapacityUnits"],
|
|
72
|
+
"ProvisionedThroughputWriteCapacityUnits": gsi[
|
|
73
|
+
"ProvisionedThroughput"
|
|
74
|
+
]["WriteCapacityUnits"],
|
|
75
|
+
"GSIName": gsi["IndexName"],
|
|
76
|
+
},
|
|
77
|
+
)
|
|
59
78
|
return ddb_table_data, ddb_gsi_data
|
|
60
79
|
|
|
61
80
|
|
|
62
81
|
@timeit
|
|
63
82
|
def load_dynamodb_tables(
|
|
64
|
-
neo4j_session: neo4j.Session,
|
|
83
|
+
neo4j_session: neo4j.Session,
|
|
84
|
+
tables_data: List[Dict[str, Any]],
|
|
85
|
+
region: str,
|
|
86
|
+
current_aws_account_id: str,
|
|
65
87
|
aws_update_tag: int,
|
|
66
88
|
) -> None:
|
|
67
|
-
logger.info(
|
|
89
|
+
logger.info(
|
|
90
|
+
f"Loading Dynamo DB tables {len(tables_data)} for region '{region}' into graph.",
|
|
91
|
+
)
|
|
68
92
|
load(
|
|
69
93
|
neo4j_session,
|
|
70
94
|
DynamoDBTableSchema(),
|
|
@@ -77,10 +101,15 @@ def load_dynamodb_tables(
|
|
|
77
101
|
|
|
78
102
|
@timeit
|
|
79
103
|
def load_dynamodb_gsi(
|
|
80
|
-
neo4j_session: neo4j.Session,
|
|
104
|
+
neo4j_session: neo4j.Session,
|
|
105
|
+
gsi_data: List[Dict[str, Any]],
|
|
106
|
+
region: str,
|
|
107
|
+
current_aws_account_id: str,
|
|
81
108
|
aws_update_tag: int,
|
|
82
109
|
) -> None:
|
|
83
|
-
logger.info(
|
|
110
|
+
logger.info(
|
|
111
|
+
f"Loading Dynamo DB GSI {len(gsi_data)} for region '{region}' into graph.",
|
|
112
|
+
)
|
|
84
113
|
load(
|
|
85
114
|
neo4j_session,
|
|
86
115
|
DynamoDBGSISchema(),
|
|
@@ -92,38 +121,77 @@ def load_dynamodb_gsi(
|
|
|
92
121
|
|
|
93
122
|
|
|
94
123
|
@timeit
|
|
95
|
-
def cleanup_dynamodb_tables(
|
|
96
|
-
|
|
97
|
-
|
|
124
|
+
def cleanup_dynamodb_tables(
|
|
125
|
+
neo4j_session: neo4j.Session,
|
|
126
|
+
common_job_parameters: Dict,
|
|
127
|
+
) -> None:
|
|
128
|
+
GraphJob.from_node_schema(DynamoDBTableSchema(), common_job_parameters).run(
|
|
129
|
+
neo4j_session,
|
|
130
|
+
)
|
|
131
|
+
GraphJob.from_node_schema(DynamoDBGSISchema(), common_job_parameters).run(
|
|
132
|
+
neo4j_session,
|
|
133
|
+
)
|
|
98
134
|
|
|
99
135
|
|
|
100
136
|
@timeit
|
|
101
137
|
def sync_dynamodb_tables(
|
|
102
|
-
neo4j_session: neo4j.Session,
|
|
103
|
-
|
|
138
|
+
neo4j_session: neo4j.Session,
|
|
139
|
+
boto3_session: boto3.session.Session,
|
|
140
|
+
regions: List[str],
|
|
141
|
+
current_aws_account_id: str,
|
|
142
|
+
aws_update_tag: int,
|
|
143
|
+
common_job_parameters: Dict,
|
|
104
144
|
) -> None:
|
|
105
145
|
for region in regions:
|
|
106
|
-
logger.info(
|
|
146
|
+
logger.info(
|
|
147
|
+
"Syncing DynamoDB for region in '%s' in account '%s'.",
|
|
148
|
+
region,
|
|
149
|
+
current_aws_account_id,
|
|
150
|
+
)
|
|
107
151
|
dynamodb_tables = get_dynamodb_tables(boto3_session, region)
|
|
108
|
-
ddb_table_data, ddb_gsi_data = transform_dynamodb_tables(
|
|
109
|
-
|
|
110
|
-
|
|
152
|
+
ddb_table_data, ddb_gsi_data = transform_dynamodb_tables(
|
|
153
|
+
dynamodb_tables,
|
|
154
|
+
region,
|
|
155
|
+
)
|
|
156
|
+
load_dynamodb_tables(
|
|
157
|
+
neo4j_session,
|
|
158
|
+
ddb_table_data,
|
|
159
|
+
region,
|
|
160
|
+
current_aws_account_id,
|
|
161
|
+
aws_update_tag,
|
|
162
|
+
)
|
|
163
|
+
load_dynamodb_gsi(
|
|
164
|
+
neo4j_session,
|
|
165
|
+
ddb_gsi_data,
|
|
166
|
+
region,
|
|
167
|
+
current_aws_account_id,
|
|
168
|
+
aws_update_tag,
|
|
169
|
+
)
|
|
111
170
|
cleanup_dynamodb_tables(neo4j_session, common_job_parameters)
|
|
112
171
|
|
|
113
172
|
|
|
114
173
|
@timeit
|
|
115
174
|
def sync(
|
|
116
|
-
neo4j_session: neo4j.Session,
|
|
117
|
-
|
|
175
|
+
neo4j_session: neo4j.Session,
|
|
176
|
+
boto3_session: boto3.session.Session,
|
|
177
|
+
regions: List[str],
|
|
178
|
+
current_aws_account_id: str,
|
|
179
|
+
update_tag: int,
|
|
180
|
+
common_job_parameters: Dict,
|
|
118
181
|
) -> None:
|
|
119
182
|
sync_dynamodb_tables(
|
|
120
|
-
neo4j_session,
|
|
183
|
+
neo4j_session,
|
|
184
|
+
boto3_session,
|
|
185
|
+
regions,
|
|
186
|
+
current_aws_account_id,
|
|
187
|
+
update_tag,
|
|
188
|
+
common_job_parameters,
|
|
121
189
|
)
|
|
122
190
|
merge_module_sync_metadata(
|
|
123
191
|
neo4j_session,
|
|
124
|
-
group_type=
|
|
192
|
+
group_type="AWSAccount",
|
|
125
193
|
group_id=current_aws_account_id,
|
|
126
|
-
synced_type=
|
|
194
|
+
synced_type="DynamoDBTable",
|
|
127
195
|
update_tag=update_tag,
|
|
128
196
|
stat_handler=stat_handler,
|
|
129
197
|
)
|
|
@@ -10,6 +10,6 @@ logger = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
@timeit
|
|
12
12
|
def get_ec2_regions(boto3_session: boto3.session.Session) -> List[str]:
|
|
13
|
-
client = boto3_session.client(
|
|
13
|
+
client = boto3_session.client("ec2")
|
|
14
14
|
result = client.describe_regions()
|
|
15
|
-
return [r[
|
|
15
|
+
return [r["RegionName"] for r in result["Regions"]]
|
|
@@ -5,20 +5,26 @@ from typing import Any
|
|
|
5
5
|
import boto3
|
|
6
6
|
import neo4j
|
|
7
7
|
|
|
8
|
-
from .util import get_botocore_config
|
|
9
8
|
from cartography.client.core.tx import load
|
|
10
9
|
from cartography.graph.job import GraphJob
|
|
11
10
|
from cartography.models.aws.ec2.auto_scaling_groups import AutoScalingGroupSchema
|
|
12
|
-
from cartography.models.aws.ec2.auto_scaling_groups import
|
|
13
|
-
|
|
11
|
+
from cartography.models.aws.ec2.auto_scaling_groups import (
|
|
12
|
+
EC2InstanceAutoScalingGroupSchema,
|
|
13
|
+
)
|
|
14
|
+
from cartography.models.aws.ec2.auto_scaling_groups import (
|
|
15
|
+
EC2SubnetAutoScalingGroupSchema,
|
|
16
|
+
)
|
|
14
17
|
from cartography.models.aws.ec2.launch_configurations import LaunchConfigurationSchema
|
|
15
18
|
from cartography.util import aws_handle_regions
|
|
16
19
|
from cartography.util import timeit
|
|
17
20
|
|
|
21
|
+
from .util import get_botocore_config
|
|
22
|
+
|
|
18
23
|
logger = logging.getLogger(__name__)
|
|
19
24
|
|
|
20
25
|
AsgData = namedtuple(
|
|
21
|
-
|
|
26
|
+
"AsgData",
|
|
27
|
+
[
|
|
22
28
|
"group_list",
|
|
23
29
|
"instance_list",
|
|
24
30
|
"subnet_list",
|
|
@@ -28,46 +34,66 @@ AsgData = namedtuple(
|
|
|
28
34
|
|
|
29
35
|
@timeit
|
|
30
36
|
@aws_handle_regions
|
|
31
|
-
def get_ec2_auto_scaling_groups(
|
|
32
|
-
|
|
33
|
-
|
|
37
|
+
def get_ec2_auto_scaling_groups(
|
|
38
|
+
boto3_session: boto3.session.Session,
|
|
39
|
+
region: str,
|
|
40
|
+
) -> list[dict]:
|
|
41
|
+
client = boto3_session.client(
|
|
42
|
+
"autoscaling",
|
|
43
|
+
region_name=region,
|
|
44
|
+
config=get_botocore_config(),
|
|
45
|
+
)
|
|
46
|
+
paginator = client.get_paginator("describe_auto_scaling_groups")
|
|
34
47
|
asgs: list[dict] = []
|
|
35
48
|
for page in paginator.paginate():
|
|
36
|
-
asgs.extend(page[
|
|
49
|
+
asgs.extend(page["AutoScalingGroups"])
|
|
37
50
|
return asgs
|
|
38
51
|
|
|
39
52
|
|
|
40
53
|
@timeit
|
|
41
54
|
@aws_handle_regions
|
|
42
|
-
def get_launch_configurations(
|
|
43
|
-
|
|
44
|
-
|
|
55
|
+
def get_launch_configurations(
|
|
56
|
+
boto3_session: boto3.session.Session,
|
|
57
|
+
region: str,
|
|
58
|
+
) -> list[dict]:
|
|
59
|
+
client = boto3_session.client(
|
|
60
|
+
"autoscaling",
|
|
61
|
+
region_name=region,
|
|
62
|
+
config=get_botocore_config(),
|
|
63
|
+
)
|
|
64
|
+
paginator = client.get_paginator("describe_launch_configurations")
|
|
45
65
|
lcs: list[dict] = []
|
|
46
66
|
for page in paginator.paginate():
|
|
47
|
-
lcs.extend(page[
|
|
67
|
+
lcs.extend(page["LaunchConfigurations"])
|
|
48
68
|
return lcs
|
|
49
69
|
|
|
50
70
|
|
|
51
|
-
def transform_launch_configurations(
|
|
71
|
+
def transform_launch_configurations(
|
|
72
|
+
configurations: list[dict[str, Any]],
|
|
73
|
+
) -> list[dict[str, Any]]:
|
|
52
74
|
transformed_configurations = []
|
|
53
75
|
for config in configurations:
|
|
54
|
-
transformed_configurations.append(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
76
|
+
transformed_configurations.append(
|
|
77
|
+
{
|
|
78
|
+
"AssociatePublicIpAddress": config.get("AssociatePublicIpAddress"),
|
|
79
|
+
"LaunchConfigurationARN": config.get("LaunchConfigurationARN"),
|
|
80
|
+
"LaunchConfigurationName": config.get("LaunchConfigurationName"),
|
|
81
|
+
"CreatedTime": config.get("CreatedTime"),
|
|
82
|
+
"ImageId": config.get("ImageId"),
|
|
83
|
+
"KeyName": config.get("KeyName"),
|
|
84
|
+
"SecurityGroups": config.get("SecurityGroups"),
|
|
85
|
+
"InstanceType": config.get("InstanceType"),
|
|
86
|
+
"KernelId": config.get("KernelId"),
|
|
87
|
+
"RamdiskId": config.get("RamdiskId"),
|
|
88
|
+
"InstanceMonitoring": config.get("InstanceMonitoring", {}).get(
|
|
89
|
+
"Enabled",
|
|
90
|
+
),
|
|
91
|
+
"SpotPrice": config.get("SpotPrice"),
|
|
92
|
+
"IamInstanceProfile": config.get("IamInstanceProfile"),
|
|
93
|
+
"EbsOptimized": config.get("EbsOptimized"),
|
|
94
|
+
"PlacementTenancy": config.get("PlacementTenancy"),
|
|
95
|
+
},
|
|
96
|
+
)
|
|
71
97
|
return transformed_configurations
|
|
72
98
|
|
|
73
99
|
|
|
@@ -76,42 +102,54 @@ def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
|
|
|
76
102
|
related_vpcs = []
|
|
77
103
|
related_instances = []
|
|
78
104
|
for group in groups:
|
|
79
|
-
transformed_groups.append(
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
105
|
+
transformed_groups.append(
|
|
106
|
+
{
|
|
107
|
+
"AutoScalingGroupARN": group["AutoScalingGroupARN"],
|
|
108
|
+
"CapacityRebalance": group.get("CapacityRebalance"),
|
|
109
|
+
"CreatedTime": str(group.get("CreatedTime")),
|
|
110
|
+
"DefaultCooldown": group.get("DefaultCooldown"),
|
|
111
|
+
"DesiredCapacity": group.get("DesiredCapacity"),
|
|
112
|
+
"HealthCheckGracePeriod": group.get("HealthCheckGracePeriod"),
|
|
113
|
+
"HealthCheckType": group.get("HealthCheckType"),
|
|
114
|
+
"LaunchConfigurationName": group.get("LaunchConfigurationName"),
|
|
115
|
+
"LaunchTemplateName": group.get("LaunchTemplate", {}).get(
|
|
116
|
+
"LaunchTemplateName",
|
|
117
|
+
),
|
|
118
|
+
"LaunchTemplateId": group.get("LaunchTemplate", {}).get(
|
|
119
|
+
"LaunchTemplateId",
|
|
120
|
+
),
|
|
121
|
+
"LaunchTemplateVersion": group.get("LaunchTemplate", {}).get("Version"),
|
|
122
|
+
"MaxInstanceLifetime": group.get("MaxInstanceLifetime"),
|
|
123
|
+
"MaxSize": group.get("MaxSize"),
|
|
124
|
+
"MinSize": group.get("MinSize"),
|
|
125
|
+
"AutoScalingGroupName": group.get("AutoScalingGroupName"),
|
|
126
|
+
"NewInstancesProtectedFromScaleIn": group.get(
|
|
127
|
+
"NewInstancesProtectedFromScaleIn",
|
|
128
|
+
),
|
|
129
|
+
"Status": group.get("Status"),
|
|
130
|
+
},
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
if group.get("VPCZoneIdentifier", None):
|
|
134
|
+
vpclist = group["VPCZoneIdentifier"]
|
|
135
|
+
subnet_ids = vpclist.split(",") if "," in vpclist else [vpclist]
|
|
102
136
|
subnets = []
|
|
103
137
|
for subnet_id in subnet_ids:
|
|
104
|
-
subnets.append(
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
138
|
+
subnets.append(
|
|
139
|
+
{
|
|
140
|
+
"VPCZoneIdentifier": subnet_id,
|
|
141
|
+
"AutoScalingGroupARN": group["AutoScalingGroupARN"],
|
|
142
|
+
},
|
|
143
|
+
)
|
|
108
144
|
related_vpcs.extend(subnets)
|
|
109
145
|
|
|
110
|
-
for instance_data in group.get(
|
|
111
|
-
related_instances.append(
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
146
|
+
for instance_data in group.get("Instances", []):
|
|
147
|
+
related_instances.append(
|
|
148
|
+
{
|
|
149
|
+
"InstanceId": instance_data["InstanceId"],
|
|
150
|
+
"AutoScalingGroupARN": group["AutoScalingGroupARN"],
|
|
151
|
+
},
|
|
152
|
+
)
|
|
115
153
|
|
|
116
154
|
return AsgData(
|
|
117
155
|
group_list=transformed_groups,
|
|
@@ -122,7 +160,11 @@ def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
|
|
|
122
160
|
|
|
123
161
|
@timeit
|
|
124
162
|
def load_launch_configurations(
|
|
125
|
-
neo4j_session: neo4j.Session,
|
|
163
|
+
neo4j_session: neo4j.Session,
|
|
164
|
+
data: list[dict],
|
|
165
|
+
region: str,
|
|
166
|
+
current_aws_account_id: str,
|
|
167
|
+
update_tag: int,
|
|
126
168
|
) -> None:
|
|
127
169
|
load(
|
|
128
170
|
neo4j_session,
|
|
@@ -135,7 +177,11 @@ def load_launch_configurations(
|
|
|
135
177
|
|
|
136
178
|
|
|
137
179
|
def load_groups(
|
|
138
|
-
|
|
180
|
+
neo4j_session: neo4j.Session,
|
|
181
|
+
data: list[dict],
|
|
182
|
+
region: str,
|
|
183
|
+
current_aws_account_id: str,
|
|
184
|
+
update_tag: int,
|
|
139
185
|
) -> None:
|
|
140
186
|
load(
|
|
141
187
|
neo4j_session,
|
|
@@ -148,7 +194,11 @@ def load_groups(
|
|
|
148
194
|
|
|
149
195
|
|
|
150
196
|
def load_asg_subnets(
|
|
151
|
-
|
|
197
|
+
neo4j_session: neo4j.Session,
|
|
198
|
+
data: list[dict],
|
|
199
|
+
region: str,
|
|
200
|
+
current_aws_account_id: str,
|
|
201
|
+
update_tag: int,
|
|
152
202
|
) -> None:
|
|
153
203
|
load(
|
|
154
204
|
neo4j_session,
|
|
@@ -161,7 +211,11 @@ def load_asg_subnets(
|
|
|
161
211
|
|
|
162
212
|
|
|
163
213
|
def load_asg_instances(
|
|
164
|
-
|
|
214
|
+
neo4j_session: neo4j.Session,
|
|
215
|
+
data: list[dict],
|
|
216
|
+
region: str,
|
|
217
|
+
current_aws_account_id: str,
|
|
218
|
+
update_tag: int,
|
|
165
219
|
) -> None:
|
|
166
220
|
load(
|
|
167
221
|
neo4j_session,
|
|
@@ -175,31 +229,80 @@ def load_asg_instances(
|
|
|
175
229
|
|
|
176
230
|
@timeit
|
|
177
231
|
def load_auto_scaling_groups(
|
|
178
|
-
neo4j_session: neo4j.Session,
|
|
232
|
+
neo4j_session: neo4j.Session,
|
|
233
|
+
data: AsgData,
|
|
234
|
+
region: str,
|
|
235
|
+
current_aws_account_id: str,
|
|
236
|
+
update_tag: int,
|
|
179
237
|
) -> None:
|
|
180
|
-
load_groups(
|
|
181
|
-
|
|
182
|
-
|
|
238
|
+
load_groups(
|
|
239
|
+
neo4j_session,
|
|
240
|
+
data.group_list,
|
|
241
|
+
region,
|
|
242
|
+
current_aws_account_id,
|
|
243
|
+
update_tag,
|
|
244
|
+
)
|
|
245
|
+
load_asg_instances(
|
|
246
|
+
neo4j_session,
|
|
247
|
+
data.instance_list,
|
|
248
|
+
region,
|
|
249
|
+
current_aws_account_id,
|
|
250
|
+
update_tag,
|
|
251
|
+
)
|
|
252
|
+
load_asg_subnets(
|
|
253
|
+
neo4j_session,
|
|
254
|
+
data.subnet_list,
|
|
255
|
+
region,
|
|
256
|
+
current_aws_account_id,
|
|
257
|
+
update_tag,
|
|
258
|
+
)
|
|
183
259
|
|
|
184
260
|
|
|
185
261
|
@timeit
|
|
186
|
-
def cleanup(
|
|
262
|
+
def cleanup(
|
|
263
|
+
neo4j_session: neo4j.Session,
|
|
264
|
+
common_job_parameters: dict[str, Any],
|
|
265
|
+
) -> None:
|
|
187
266
|
logger.debug("Running EC2 instance cleanup")
|
|
188
|
-
GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(
|
|
189
|
-
|
|
267
|
+
GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(
|
|
268
|
+
neo4j_session,
|
|
269
|
+
)
|
|
270
|
+
GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(
|
|
271
|
+
neo4j_session,
|
|
272
|
+
)
|
|
190
273
|
|
|
191
274
|
|
|
192
275
|
@timeit
|
|
193
276
|
def sync_ec2_auto_scaling_groups(
|
|
194
|
-
|
|
195
|
-
|
|
277
|
+
neo4j_session: neo4j.Session,
|
|
278
|
+
boto3_session: boto3.session.Session,
|
|
279
|
+
regions: list[str],
|
|
280
|
+
current_aws_account_id: str,
|
|
281
|
+
update_tag: int,
|
|
282
|
+
common_job_parameters: dict,
|
|
196
283
|
) -> None:
|
|
197
284
|
for region in regions:
|
|
198
|
-
logger.debug(
|
|
285
|
+
logger.debug(
|
|
286
|
+
"Syncing auto scaling groups for region '%s' in account '%s'.",
|
|
287
|
+
region,
|
|
288
|
+
current_aws_account_id,
|
|
289
|
+
)
|
|
199
290
|
lc_data = get_launch_configurations(boto3_session, region)
|
|
200
291
|
asg_data = get_ec2_auto_scaling_groups(boto3_session, region)
|
|
201
292
|
lc_transformed = transform_launch_configurations(lc_data)
|
|
202
293
|
asg_transformed = transform_auto_scaling_groups(asg_data)
|
|
203
|
-
load_launch_configurations(
|
|
204
|
-
|
|
294
|
+
load_launch_configurations(
|
|
295
|
+
neo4j_session,
|
|
296
|
+
lc_transformed,
|
|
297
|
+
region,
|
|
298
|
+
current_aws_account_id,
|
|
299
|
+
update_tag,
|
|
300
|
+
)
|
|
301
|
+
load_auto_scaling_groups(
|
|
302
|
+
neo4j_session,
|
|
303
|
+
asg_transformed,
|
|
304
|
+
region,
|
|
305
|
+
current_aws_account_id,
|
|
306
|
+
update_tag,
|
|
307
|
+
)
|
|
205
308
|
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -6,20 +6,28 @@ import boto3
|
|
|
6
6
|
import neo4j
|
|
7
7
|
from botocore.exceptions import ClientError
|
|
8
8
|
|
|
9
|
-
from .util import get_botocore_config
|
|
10
9
|
from cartography.util import aws_handle_regions
|
|
11
10
|
from cartography.util import run_cleanup_job
|
|
12
11
|
from cartography.util import timeit
|
|
13
12
|
|
|
13
|
+
from .util import get_botocore_config
|
|
14
|
+
|
|
14
15
|
logger = logging.getLogger(__name__)
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
@timeit
|
|
18
19
|
@aws_handle_regions
|
|
19
|
-
def get_elastic_ip_addresses(
|
|
20
|
-
|
|
20
|
+
def get_elastic_ip_addresses(
|
|
21
|
+
boto3_session: boto3.session.Session,
|
|
22
|
+
region: str,
|
|
23
|
+
) -> List[Dict]:
|
|
24
|
+
client = boto3_session.client(
|
|
25
|
+
"ec2",
|
|
26
|
+
region_name=region,
|
|
27
|
+
config=get_botocore_config(),
|
|
28
|
+
)
|
|
21
29
|
try:
|
|
22
|
-
addresses = client.describe_addresses()[
|
|
30
|
+
addresses = client.describe_addresses()["Addresses"]
|
|
23
31
|
except ClientError as e:
|
|
24
32
|
logger.warning(f"Failed retrieve address for region - {region}. Error - {e}")
|
|
25
33
|
raise
|
|
@@ -28,8 +36,11 @@ def get_elastic_ip_addresses(boto3_session: boto3.session.Session, region: str)
|
|
|
28
36
|
|
|
29
37
|
@timeit
|
|
30
38
|
def load_elastic_ip_addresses(
|
|
31
|
-
neo4j_session: neo4j.Session,
|
|
32
|
-
|
|
39
|
+
neo4j_session: neo4j.Session,
|
|
40
|
+
elastic_ip_addresses: List[Dict],
|
|
41
|
+
region: str,
|
|
42
|
+
current_aws_account_id: str,
|
|
43
|
+
update_tag: int,
|
|
33
44
|
) -> None:
|
|
34
45
|
"""
|
|
35
46
|
Creates (:ElasticIpAddress)
|
|
@@ -37,7 +48,9 @@ def load_elastic_ip_addresses(
|
|
|
37
48
|
(:EC2Instance)-[:ELASTIC_IP_ADDRESS]->(:ElasticIpAddress),
|
|
38
49
|
(:NetworkInterface)-[:ELASTIC_IP_ADDRESS]->(:ElasticIpAddress),
|
|
39
50
|
"""
|
|
40
|
-
logger.info(
|
|
51
|
+
logger.info(
|
|
52
|
+
f"Loading {len(elastic_ip_addresses)} Elastic IP Addresses in {region}.",
|
|
53
|
+
)
|
|
41
54
|
ingest_addresses = """
|
|
42
55
|
UNWIND $elastic_ip_addresses as eia
|
|
43
56
|
MERGE (address: ElasticIPAddress{id: eia.PublicIp})
|
|
@@ -80,9 +93,12 @@ def load_elastic_ip_addresses(
|
|
|
80
93
|
|
|
81
94
|
|
|
82
95
|
@timeit
|
|
83
|
-
def cleanup_elastic_ip_addresses(
|
|
96
|
+
def cleanup_elastic_ip_addresses(
|
|
97
|
+
neo4j_session: neo4j.Session,
|
|
98
|
+
common_job_parameters: Dict,
|
|
99
|
+
) -> None:
|
|
84
100
|
run_cleanup_job(
|
|
85
|
-
|
|
101
|
+
"aws_import_elastic_ip_addresses_cleanup.json",
|
|
86
102
|
neo4j_session,
|
|
87
103
|
common_job_parameters,
|
|
88
104
|
)
|
|
@@ -90,11 +106,23 @@ def cleanup_elastic_ip_addresses(neo4j_session: neo4j.Session, common_job_parame
|
|
|
90
106
|
|
|
91
107
|
@timeit
|
|
92
108
|
def sync_elastic_ip_addresses(
|
|
93
|
-
neo4j_session: neo4j.Session,
|
|
94
|
-
|
|
109
|
+
neo4j_session: neo4j.Session,
|
|
110
|
+
boto3_session: boto3.session.Session,
|
|
111
|
+
regions: List[str],
|
|
112
|
+
current_aws_account_id: str,
|
|
113
|
+
update_tag: int,
|
|
114
|
+
common_job_parameters: Dict,
|
|
95
115
|
) -> None:
|
|
96
116
|
for region in regions:
|
|
97
|
-
logger.info(
|
|
117
|
+
logger.info(
|
|
118
|
+
f"Syncing Elastic IP Addresses for region {region} in account {current_aws_account_id}.",
|
|
119
|
+
)
|
|
98
120
|
addresses = get_elastic_ip_addresses(boto3_session, region)
|
|
99
|
-
load_elastic_ip_addresses(
|
|
121
|
+
load_elastic_ip_addresses(
|
|
122
|
+
neo4j_session,
|
|
123
|
+
addresses,
|
|
124
|
+
region,
|
|
125
|
+
current_aws_account_id,
|
|
126
|
+
update_tag,
|
|
127
|
+
)
|
|
100
128
|
cleanup_elastic_ip_addresses(neo4j_session, common_job_parameters)
|