cartography 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

cartography/_version.py CHANGED
@@ -1,8 +1,13 @@
1
- # file generated by setuptools_scm
1
+ # file generated by setuptools-scm
2
2
  # don't change, don't track in version control
3
+
4
+ __all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
5
+
3
6
  TYPE_CHECKING = False
4
7
  if TYPE_CHECKING:
5
- from typing import Tuple, Union
8
+ from typing import Tuple
9
+ from typing import Union
10
+
6
11
  VERSION_TUPLE = Tuple[Union[int, str], ...]
7
12
  else:
8
13
  VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
12
17
  __version_tuple__: VERSION_TUPLE
13
18
  version_tuple: VERSION_TUPLE
14
19
 
15
- __version__ = version = '0.99.0'
16
- __version_tuple__ = version_tuple = (0, 99, 0)
20
+ __version__ = version = '0.100.0'
21
+ __version_tuple__ = version_tuple = (0, 100, 0)
cartography/cli.py CHANGED
@@ -439,9 +439,10 @@ class CLI:
439
439
  '--gsuite-auth-method',
440
440
  type=str,
441
441
  default='delegated',
442
- choices=['delegated', 'oauth'],
442
+ choices=['delegated', 'oauth', 'default'],
443
443
  help=(
444
- 'The method used by GSuite to authenticate. delegated is the legacy one.'
444
+ 'GSuite authentication method. Can be "delegated" for service account or "oauth" for OAuth. '
445
+ '"Default" best if using gcloud CLI.'
445
446
  ),
446
447
  )
447
448
  parser.add_argument(
@@ -12,8 +12,13 @@ import neo4j
12
12
  from botocore.exceptions import ClientError
13
13
  from policyuniverse.policy import Policy
14
14
 
15
+ from cartography.client.core.tx import load
16
+ from cartography.graph.job import GraphJob
17
+ from cartography.models.aws.apigateway import APIGatewayRestAPISchema
18
+ from cartography.models.aws.apigatewaycertificate import APIGatewayClientCertificateSchema
19
+ from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
20
+ from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
15
21
  from cartography.util import aws_handle_regions
16
- from cartography.util import run_cleanup_job
17
22
  from cartography.util import timeit
18
23
 
19
24
  logger = logging.getLogger(__name__)
@@ -107,222 +112,146 @@ def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
107
112
  return policy
108
113
 
109
114
 
110
- @timeit
111
- def load_apigateway_rest_apis(
112
- neo4j_session: neo4j.Session, rest_apis: List[Dict], region: str, current_aws_account_id: str,
113
- aws_update_tag: int,
114
- ) -> None:
115
- """
116
- Ingest the details of API Gateway REST APIs into neo4j.
115
+ def transform_apigateway_rest_apis(
116
+ rest_apis: List[Dict], resource_policies: List[Dict], region: str, current_aws_account_id: str, aws_update_tag: int,
117
+ ) -> List[Dict]:
117
118
  """
118
- ingest_rest_apis = """
119
- UNWIND $rest_apis_list AS r
120
- MERGE (rest_api:APIGatewayRestAPI{id:r.id})
121
- ON CREATE SET rest_api.firstseen = timestamp(),
122
- rest_api.createddate = r.createdDate
123
- SET rest_api.version = r.version,
124
- rest_api.minimumcompressionsize = r.minimumCompressionSize,
125
- rest_api.disableexecuteapiendpoint = r.disableExecuteApiEndpoint,
126
- rest_api.lastupdated = $aws_update_tag,
127
- rest_api.region = $Region
128
- WITH rest_api
129
- MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
130
- MERGE (aa)-[r:RESOURCE]->(rest_api)
131
- ON CREATE SET r.firstseen = timestamp()
132
- SET r.lastupdated = $aws_update_tag
119
+ Transform API Gateway REST API data for ingestion, including policy analysis
133
120
  """
121
+ # Create a mapping of api_id to policy data for easier lookup
122
+ policy_map = {
123
+ policy['api_id']: policy
124
+ for policy in resource_policies
125
+ }
134
126
 
135
- # neo4j does not accept datetime objects and values. This loop is used to convert
136
- # these values to string.
127
+ transformed_apis = []
137
128
  for api in rest_apis:
138
- api['createdDate'] = str(api['createdDate']) if 'createdDate' in api else None
139
-
140
- neo4j_session.run(
141
- ingest_rest_apis,
142
- rest_apis_list=rest_apis,
143
- aws_update_tag=aws_update_tag,
144
- Region=region,
145
- AWS_ACCOUNT_ID=current_aws_account_id,
146
- )
129
+ policy_data = policy_map.get(api['id'], {})
130
+ transformed_api = {
131
+ 'id': api['id'],
132
+ 'createdDate': str(api['createdDate']) if 'createdDate' in api else None,
133
+ 'version': api.get('version'),
134
+ 'minimumCompressionSize': api.get('minimumCompressionSize'),
135
+ 'disableExecuteApiEndpoint': api.get('disableExecuteApiEndpoint'),
136
+ # Set defaults in the transform function
137
+ 'anonymous_access': policy_data.get('internet_accessible', False),
138
+ 'anonymous_actions': policy_data.get('accessible_actions', []),
139
+ # TODO Issue #1452: clarify internet exposure vs anonymous access
140
+ }
141
+ transformed_apis.append(transformed_api)
142
+
143
+ return transformed_apis
147
144
 
148
145
 
149
146
  @timeit
150
- def _load_apigateway_policies(
151
- neo4j_session: neo4j.Session, policies: List, update_tag: int,
147
+ def load_apigateway_rest_apis(
148
+ neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str,
149
+ aws_update_tag: int,
152
150
  ) -> None:
153
151
  """
154
- Ingest API Gateway REST API policy results into neo4j.
155
- """
156
- ingest_policies = """
157
- UNWIND $policies as policy
158
- MATCH (r:APIGatewayRestAPI) where r.name = policy.api_id
159
- SET r.anonymous_access = (coalesce(r.anonymous_access, false) OR policy.internet_accessible),
160
- r.anonymous_actions = coalesce(r.anonymous_actions, []) + policy.accessible_actions,
161
- r.lastupdated = $UpdateTag
152
+ Ingest API Gateway REST API data into neo4j.
162
153
  """
163
-
164
- neo4j_session.run(
165
- ingest_policies,
166
- policies=policies,
167
- UpdateTag=update_tag,
168
- )
169
-
170
-
171
- def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
172
- set_defaults = """
173
- MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(restApi:APIGatewayRestAPI)
174
- where restApi.anonymous_actions IS NULL
175
- SET restApi.anonymous_access = false, restApi.anonymous_actions = []
176
- """
177
-
178
- neo4j_session.run(
179
- set_defaults,
180
- AWS_ID=aws_account_id,
154
+ load(
155
+ neo4j_session,
156
+ APIGatewayRestAPISchema(),
157
+ data,
158
+ region=region,
159
+ lastupdated=aws_update_tag,
160
+ AWS_ID=current_aws_account_id,
181
161
  )
182
162
 
183
163
 
184
- @timeit
185
- def _load_apigateway_stages(
186
- neo4j_session: neo4j.Session, stages: List, update_tag: int,
187
- ) -> None:
164
+ def transform_apigateway_stages(stages: List[Dict], update_tag: int) -> List[Dict]:
188
165
  """
189
- Ingest the Stage resource details into neo4j.
166
+ Transform API Gateway Stage data for ingestion
190
167
  """
191
- ingest_stages = """
192
- UNWIND $stages_list AS stage
193
- MERGE (s:APIGatewayStage{id: stage.arn})
194
- ON CREATE SET s.firstseen = timestamp(), s.stagename = stage.stageName,
195
- s.createddate = stage.createdDate
196
- SET s.deploymentid = stage.deploymentId,
197
- s.clientcertificateid = stage.clientCertificateId,
198
- s.cacheclusterenabled = stage.cacheClusterEnabled,
199
- s.cacheclusterstatus = stage.cacheClusterStatus,
200
- s.tracingenabled = stage.tracingEnabled,
201
- s.webaclarn = stage.webAclArn,
202
- s.lastupdated = $UpdateTag
203
- WITH s, stage
204
- MATCH (rest_api:APIGatewayRestAPI{id: stage.apiId})
205
- MERGE (rest_api)-[r:ASSOCIATED_WITH]->(s)
206
- ON CREATE SET r.firstseen = timestamp()
207
- SET r.lastupdated = $UpdateTag
208
- """
209
-
210
- # neo4j does not accept datetime objects and values. This loop is used to convert
211
- # these values to string.
168
+ stage_data = []
212
169
  for stage in stages:
213
170
  stage['createdDate'] = str(stage['createdDate'])
214
- stage['arn'] = "arn:aws:apigateway:::" + stage['apiId'] + "/" + stage['stageName']
215
-
216
- neo4j_session.run(
217
- ingest_stages,
218
- stages_list=stages,
219
- UpdateTag=update_tag,
220
- )
171
+ stage['arn'] = f"arn:aws:apigateway:::{stage['apiId']}/{stage['stageName']}"
172
+ stage_data.append(stage)
173
+ return stage_data
221
174
 
222
175
 
223
- @timeit
224
- def _load_apigateway_certificates(
225
- neo4j_session: neo4j.Session, certificates: List, update_tag: int,
226
- ) -> None:
227
- """
228
- Ingest the API Gateway Client Certificate details into neo4j.
176
+ def transform_apigateway_certificates(certificates: List[Dict], update_tag: int) -> List[Dict]:
229
177
  """
230
- ingest_certificates = """
231
- UNWIND $certificates_list as certificate
232
- MERGE (c:APIGatewayClientCertificate{id: certificate.clientCertificateId})
233
- ON CREATE SET c.firstseen = timestamp(), c.createddate = certificate.createdDate
234
- SET c.lastupdated = $UpdateTag, c.expirationdate = certificate.expirationDate
235
- WITH c, certificate
236
- MATCH (stage:APIGatewayStage{id: certificate.stageArn})
237
- MERGE (stage)-[r:HAS_CERTIFICATE]->(c)
238
- ON CREATE SET r.firstseen = timestamp()
239
- SET r.lastupdated = $UpdateTag
178
+ Transform API Gateway Client Certificate data for ingestion
240
179
  """
241
-
242
- # neo4j does not accept datetime objects and values. This loop is used to convert
243
- # these values to string.
180
+ cert_data = []
244
181
  for certificate in certificates:
245
182
  certificate['createdDate'] = str(certificate['createdDate'])
246
183
  certificate['expirationDate'] = str(certificate.get('expirationDate'))
247
- certificate['stageArn'] = "arn:aws:apigateway:::" + certificate['apiId'] + "/" + certificate['stageName']
248
-
249
- neo4j_session.run(
250
- ingest_certificates,
251
- certificates_list=certificates,
252
- UpdateTag=update_tag,
253
- )
254
-
255
-
256
- @timeit
257
- def _load_apigateway_resources(
258
- neo4j_session: neo4j.Session, resources: List, update_tag: int,
259
- ) -> None:
260
- """
261
- Ingest the API Gateway Resource details into neo4j.
262
- """
263
- ingest_resources = """
264
- UNWIND $resources_list AS res
265
- MERGE (s:APIGatewayResource{id: res.id})
266
- ON CREATE SET s.firstseen = timestamp()
267
- SET s.path = res.path,
268
- s.pathpart = res.pathPart,
269
- s.parentid = res.parentId,
270
- s.lastupdated =$UpdateTag
271
- WITH s, res
272
- MATCH (rest_api:APIGatewayRestAPI{id: res.apiId})
273
- MERGE (rest_api)-[r:RESOURCE]->(s)
274
- ON CREATE SET r.firstseen = timestamp()
275
- SET r.lastupdated = $UpdateTag
276
- """
277
-
278
- neo4j_session.run(
279
- ingest_resources,
280
- resources_list=resources,
281
- UpdateTag=update_tag,
282
- )
184
+ certificate['stageArn'] = f"arn:aws:apigateway:::{certificate['apiId']}/{certificate['stageName']}"
185
+ cert_data.append(certificate)
186
+ return cert_data
283
187
 
284
188
 
285
- @timeit
286
- def load_rest_api_details(
287
- neo4j_session: neo4j.Session, stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
288
- aws_account_id: str, update_tag: int,
289
- ) -> None:
189
+ def transform_rest_api_details(
190
+ stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
191
+ ) -> Tuple[List[Dict], List[Dict], List[Dict]]:
290
192
  """
291
- Create dictionaries for Stages, Client certificates, policies and Resource resources
292
- so we can import them in a single query
193
+ Transform Stage, Client Certificate, and Resource data for ingestion
293
194
  """
294
195
  stages: List[Dict] = []
295
196
  certificates: List[Dict] = []
296
197
  resources: List[Dict] = []
297
- policies: List = []
298
- for api_id, stage, certificate, resource, policy in stages_certificate_resources:
299
- parsed_policy = parse_policy(api_id, policy)
300
- if parsed_policy is not None:
301
- policies.append(parsed_policy)
198
+
199
+ for api_id, stage, certificate, resource, _ in stages_certificate_resources:
302
200
  if len(stage) > 0:
303
201
  for s in stage:
304
202
  s['apiId'] = api_id
203
+ s['createdDate'] = str(s['createdDate'])
204
+ s['arn'] = f"arn:aws:apigateway:::{api_id}/{s['stageName']}"
305
205
  stages.extend(stage)
206
+
207
+ if certificate:
208
+ certificate['apiId'] = api_id
209
+ certificate['createdDate'] = str(certificate['createdDate'])
210
+ certificate['expirationDate'] = str(certificate.get('expirationDate'))
211
+ certificate['stageArn'] = f"arn:aws:apigateway:::{api_id}/{certificate['stageName']}"
212
+ certificates.append(certificate)
213
+
306
214
  if len(resource) > 0:
307
215
  for r in resource:
308
216
  r['apiId'] = api_id
309
217
  resources.extend(resource)
310
- if certificate:
311
- certificate['apiId'] = api_id
312
- certificates.append(certificate)
313
218
 
314
- # cleanup existing properties
315
- run_cleanup_job(
316
- 'aws_apigateway_details.json',
219
+ return stages, certificates, resources
220
+
221
+
222
+ @timeit
223
+ def load_rest_api_details(
224
+ neo4j_session: neo4j.Session, stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
225
+ aws_account_id: str, update_tag: int,
226
+ ) -> None:
227
+ """
228
+ Transform and load Stage, Client Certificate, and Resource data
229
+ """
230
+ stages, certificates, resources = transform_rest_api_details(stages_certificate_resources)
231
+
232
+ load(
317
233
  neo4j_session,
318
- {'UPDATE_TAG': update_tag, 'AWS_ID': aws_account_id},
234
+ APIGatewayStageSchema(),
235
+ stages,
236
+ lastupdated=update_tag,
237
+ AWS_ID=aws_account_id,
319
238
  )
320
239
 
321
- _load_apigateway_policies(neo4j_session, policies, update_tag)
322
- _load_apigateway_stages(neo4j_session, stages, update_tag)
323
- _load_apigateway_certificates(neo4j_session, certificates, update_tag)
324
- _load_apigateway_resources(neo4j_session, resources, update_tag)
325
- _set_default_values(neo4j_session, aws_account_id)
240
+ load(
241
+ neo4j_session,
242
+ APIGatewayClientCertificateSchema(),
243
+ certificates,
244
+ lastupdated=update_tag,
245
+ AWS_ID=aws_account_id,
246
+ )
247
+
248
+ load(
249
+ neo4j_session,
250
+ APIGatewayResourceSchema(),
251
+ resources,
252
+ lastupdated=update_tag,
253
+ AWS_ID=aws_account_id,
254
+ )
326
255
 
327
256
 
328
257
  @timeit
@@ -353,7 +282,27 @@ def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
353
282
 
354
283
  @timeit
355
284
  def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
356
- run_cleanup_job('aws_import_apigateway_cleanup.json', neo4j_session, common_job_parameters)
285
+ """
286
+ Delete out-of-date API Gateway resources and relationships.
287
+ Order matters - clean up certificates, stages, and resources before cleaning up the REST APIs they connect to.
288
+ """
289
+ logger.info("Running API Gateway cleanup job.")
290
+
291
+ # Clean up certificates first
292
+ cleanup_job = GraphJob.from_node_schema(APIGatewayClientCertificateSchema(), common_job_parameters)
293
+ cleanup_job.run(neo4j_session)
294
+
295
+ # Then stages
296
+ cleanup_job = GraphJob.from_node_schema(APIGatewayStageSchema(), common_job_parameters)
297
+ cleanup_job.run(neo4j_session)
298
+
299
+ # Then resources
300
+ cleanup_job = GraphJob.from_node_schema(APIGatewayResourceSchema(), common_job_parameters)
301
+ cleanup_job.run(neo4j_session)
302
+
303
+ # Finally REST APIs
304
+ cleanup_job = GraphJob.from_node_schema(APIGatewayRestAPISchema(), common_job_parameters)
305
+ cleanup_job.run(neo4j_session)
357
306
 
358
307
 
359
308
  @timeit
@@ -362,9 +311,23 @@ def sync_apigateway_rest_apis(
362
311
  aws_update_tag: int,
363
312
  ) -> None:
364
313
  rest_apis = get_apigateway_rest_apis(boto3_session, region)
365
- load_apigateway_rest_apis(neo4j_session, rest_apis, region, current_aws_account_id, aws_update_tag)
366
-
367
314
  stages_certificate_resources = get_rest_api_details(boto3_session, rest_apis, region)
315
+
316
+ # Extract policies and transform the data
317
+ policies = []
318
+ for api_id, _, _, _, policy in stages_certificate_resources:
319
+ parsed_policy = parse_policy(api_id, policy)
320
+ if parsed_policy is not None:
321
+ policies.append(parsed_policy)
322
+
323
+ transformed_apis = transform_apigateway_rest_apis(
324
+ rest_apis,
325
+ policies,
326
+ region,
327
+ current_aws_account_id,
328
+ aws_update_tag,
329
+ )
330
+ load_apigateway_rest_apis(neo4j_session, transformed_apis, region, current_aws_account_id, aws_update_tag)
368
331
  load_rest_api_details(neo4j_session, stages_certificate_resources, current_aws_account_id, aws_update_tag)
369
332
 
370
333
 
@@ -8,6 +8,7 @@ import neo4j
8
8
  from botocore.exceptions import ClientError
9
9
 
10
10
  from cartography.client.core.tx import load
11
+ from cartography.client.core.tx import read_list_of_values_tx
11
12
  from cartography.graph.job import GraphJob
12
13
  from cartography.intel.aws.ec2.util import get_botocore_config
13
14
  from cartography.models.aws.ec2.images import EC2ImageSchema
@@ -20,22 +21,26 @@ logger = logging.getLogger(__name__)
20
21
  @timeit
21
22
  def get_images_in_use(neo4j_session: neo4j.Session, region: str, current_aws_account_id: str) -> List[str]:
22
23
  get_images_query = """
24
+ CALL {
23
25
  MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
24
- WHERE i.region = $Region
25
- RETURN DISTINCT(i.imageid) as image
26
- UNION
26
+ WHERE i.region = $Region AND i.imageid IS NOT NULL
27
+ RETURN i.imageid AS image
28
+ UNION ALL
27
29
  MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(lc:LaunchConfiguration)
28
- WHERE lc.region = $Region
29
- RETURN DISTINCT(lc.image_id) as image
30
- UNION
30
+ WHERE lc.region = $Region AND lc.image_id IS NOT NULL
31
+ RETURN lc.image_id AS image
32
+ UNION ALL
31
33
  MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(ltv:LaunchTemplateVersion)
32
- WHERE ltv.region = $Region
33
- RETURN DISTINCT(ltv.image_id) as image
34
+ WHERE ltv.region = $Region AND ltv.image_id IS NOT NULL
35
+ RETURN ltv.image_id AS image
36
+ }
37
+ RETURN DISTINCT image;
34
38
  """
35
- results = neo4j_session.run(get_images_query, AWS_ACCOUNT_ID=current_aws_account_id, Region=region)
36
- images = []
37
- for r in results:
38
- images.append(r['image'])
39
+ result = read_list_of_values_tx(
40
+ neo4j_session, get_images_query,
41
+ AWS_ACCOUNT_ID=current_aws_account_id, Region=region,
42
+ )
43
+ images = [str(image) for image in result]
39
44
  return images
40
45
 
41
46
 
@@ -44,22 +49,23 @@ def get_images_in_use(neo4j_session: neo4j.Session, region: str, current_aws_acc
44
49
  def get_images(boto3_session: boto3.session.Session, region: str, image_ids: List[str]) -> List[Dict]:
45
50
  client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
46
51
  images = []
52
+ self_images = []
47
53
  try:
48
54
  self_images = client.describe_images(Owners=['self'])['Images']
49
- images.extend(self_images)
50
- except ClientError as e:
51
- logger.warning(f"Failed retrieve images for region - {region}. Error - {e}")
52
- try:
53
- if image_ids:
54
- image_ids = [image_id for image_id in image_ids if image_id is not None]
55
- images_in_use = client.describe_images(ImageIds=image_ids)['Images']
56
- # Ensure we're not adding duplicates
57
- _ids = [image["ImageId"] for image in images]
58
- for image in images_in_use:
59
- if image["ImageId"] not in _ids:
60
- images.append(image)
61
55
  except ClientError as e:
62
- logger.warning(f"Failed retrieve images for region - {region}. Error - {e}")
56
+ logger.warning(f"Failed retrieve self owned images for region - {region}. Error - {e}")
57
+ images.extend(self_images)
58
+ if image_ids:
59
+ self_image_ids = {image['ImageId'] for image in images}
60
+ # Go one by one to avoid losing all images if one fails
61
+ for image in image_ids:
62
+ if image in self_image_ids:
63
+ continue
64
+ try:
65
+ public_images = client.describe_images(ImageIds=[image])['Images']
66
+ images.extend(public_images)
67
+ except ClientError as e:
68
+ logger.warning(f"Failed retrieve image id {image} for region - {region}. Error - {e}")
63
69
  return images
64
70
 
65
71
 
@@ -3,7 +3,6 @@ from typing import Any
3
3
 
4
4
  import boto3
5
5
  import neo4j
6
- from botocore.exceptions import ClientError
7
6
 
8
7
  from .util import get_botocore_config
9
8
  from cartography.client.core.tx import load
@@ -21,27 +20,45 @@ logger = logging.getLogger(__name__)
21
20
  def get_launch_templates(
22
21
  boto3_session: boto3.session.Session,
23
22
  region: str,
24
- ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
23
+ ) -> list[dict[str, Any]]:
25
24
  client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
26
25
  paginator = client.get_paginator('describe_launch_templates')
27
26
  templates: list[dict[str, Any]] = []
28
- template_versions: list[dict[str, Any]] = []
29
27
  for page in paginator.paginate():
30
28
  paginated_templates = page['LaunchTemplates']
31
- for template in paginated_templates:
32
- template_id = template['LaunchTemplateId']
33
- try:
34
- versions = get_launch_template_versions_by_template(boto3_session, template_id, region)
35
- except ClientError as e:
36
- logger.warning(
37
- f"Failed to get launch template versions for {template_id}: {e}",
38
- exc_info=True,
39
- )
40
- versions = []
41
- # Using a key not defined in latest boto3 documentation
42
- template_versions.extend(versions)
43
29
  templates.extend(paginated_templates)
44
- return templates, template_versions
30
+ return templates
31
+
32
+
33
+ @timeit
34
+ @aws_handle_regions
35
+ def get_launch_template_versions(
36
+ boto3_session: boto3.session.Session,
37
+ region: str,
38
+ launch_templates: list[dict[str, Any]],
39
+ ) -> list[dict[str, Any]]:
40
+ template_versions: list[dict[str, Any]] = []
41
+ for template in launch_templates:
42
+ launch_template_id = template['LaunchTemplateId']
43
+ versions = get_launch_template_versions_by_template(boto3_session, launch_template_id, region)
44
+ template_versions.extend(versions)
45
+
46
+ return template_versions
47
+
48
+
49
+ @timeit
50
+ @aws_handle_regions
51
+ def get_launch_template_versions_by_template(
52
+ boto3_session: boto3.session.Session,
53
+ launch_template_id: str,
54
+ region: str,
55
+ ) -> list[dict[str, Any]]:
56
+ client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
57
+ v_paginator = client.get_paginator('describe_launch_template_versions')
58
+ template_versions = []
59
+ for versions in v_paginator.paginate(LaunchTemplateId=launch_template_id):
60
+ template_versions.extend(versions['LaunchTemplateVersions'])
61
+ return template_versions
45
62
 
46
63
 
47
64
  def transform_launch_templates(templates: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -71,21 +88,6 @@ def load_launch_templates(
71
88
  )
72
89
 
73
90
 
74
- @timeit
75
- @aws_handle_regions
76
- def get_launch_template_versions_by_template(
77
- boto3_session: boto3.session.Session,
78
- template: str,
79
- region: str,
80
- ) -> list[dict[str, Any]]:
81
- client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
82
- v_paginator = client.get_paginator('describe_launch_template_versions')
83
- template_versions = []
84
- for versions in v_paginator.paginate(LaunchTemplateId=template):
85
- template_versions.extend(versions['LaunchTemplateVersions'])
86
- return template_versions
87
-
88
-
89
91
  def transform_launch_template_versions(versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
90
92
  result: list[dict[str, Any]] = []
91
93
  for version in versions:
@@ -153,7 +155,8 @@ def sync_ec2_launch_templates(
153
155
  ) -> None:
154
156
  for region in regions:
155
157
  logger.info(f"Syncing launch templates for region '{region}' in account '{current_aws_account_id}'.")
156
- templates, versions = get_launch_templates(boto3_session, region)
158
+ templates = get_launch_templates(boto3_session, region)
159
+ versions = get_launch_template_versions(boto3_session, region, templates)
157
160
  templates = transform_launch_templates(templates)
158
161
  load_launch_templates(neo4j_session, templates, region, current_aws_account_id, update_tag)
159
162
  versions = transform_launch_template_versions(versions)
@@ -33,10 +33,21 @@ def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> L
33
33
  def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]:
34
34
  logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region)
35
35
  client = boto3_session.client('ecr', region_name=region)
36
- paginator = client.get_paginator('list_images')
36
+ list_paginator = client.get_paginator('list_images')
37
37
  ecr_repository_images: List[Dict] = []
38
- for page in paginator.paginate(repositoryName=repository_name):
39
- ecr_repository_images.extend(page['imageIds'])
38
+ for page in list_paginator.paginate(repositoryName=repository_name):
39
+ image_ids = page['imageIds']
40
+ if not image_ids:
41
+ continue
42
+ describe_paginator = client.get_paginator('describe_images')
43
+ describe_response = describe_paginator.paginate(repositoryName=repository_name, imageIds=image_ids)
44
+ for response in describe_response:
45
+ image_details = response['imageDetails']
46
+ image_details = [
47
+ {**detail, 'imageTag': detail['imageTags'][0]} if detail.get('imageTags') else detail
48
+ for detail in image_details
49
+ ]
50
+ ecr_repository_images.extend(image_details)
40
51
  return ecr_repository_images
41
52
 
42
53
 
@@ -103,7 +114,12 @@ def _load_ecr_repo_img_tx(
103
114
  ON CREATE SET ri.firstseen = timestamp()
104
115
  SET ri.lastupdated = $aws_update_tag,
105
116
  ri.tag = repo_img.imageTag,
106
- ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')
117
+ ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, ''),
118
+ ri.image_size_bytes = repo_img.imageSizeInBytes,
119
+ ri.image_pushed_at = repo_img.imagePushedAt,
120
+ ri.image_manifest_media_type = repo_img.imageManifestMediaType,
121
+ ri.artifact_media_type = repo_img.artifactMediaType,
122
+ ri.last_recorded_pull_time = repo_img.lastRecordedPullTime
107
123
  WITH ri, repo_img
108
124
 
109
125
  MERGE (img:ECRImage{id: repo_img.imageDigest})