cartography 0.108.0rc2__py3-none-any.whl → 0.109.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (70) hide show
  1. cartography/_version.py +2 -2
  2. cartography/cli.py +14 -0
  3. cartography/config.py +4 -0
  4. cartography/data/indexes.cypher +0 -17
  5. cartography/data/jobs/cleanup/gcp_compute_vpc_cleanup.json +0 -12
  6. cartography/intel/aws/cloudtrail_management_events.py +57 -3
  7. cartography/intel/aws/ecr.py +55 -80
  8. cartography/intel/aws/eventbridge.py +91 -0
  9. cartography/intel/aws/glue.py +117 -0
  10. cartography/intel/aws/identitycenter.py +71 -23
  11. cartography/intel/aws/kms.py +160 -200
  12. cartography/intel/aws/lambda_function.py +206 -190
  13. cartography/intel/aws/rds.py +243 -458
  14. cartography/intel/aws/resourcegroupstaggingapi.py +77 -18
  15. cartography/intel/aws/resources.py +4 -0
  16. cartography/intel/aws/route53.py +334 -332
  17. cartography/intel/aws/secretsmanager.py +62 -44
  18. cartography/intel/entra/groups.py +29 -1
  19. cartography/intel/gcp/__init__.py +10 -0
  20. cartography/intel/gcp/compute.py +19 -42
  21. cartography/intel/trivy/__init__.py +73 -13
  22. cartography/intel/trivy/scanner.py +115 -92
  23. cartography/models/aws/ecr/__init__.py +0 -0
  24. cartography/models/aws/ecr/image.py +41 -0
  25. cartography/models/aws/ecr/repository.py +72 -0
  26. cartography/models/aws/ecr/repository_image.py +95 -0
  27. cartography/models/aws/eventbridge/__init__.py +0 -0
  28. cartography/models/aws/eventbridge/rule.py +77 -0
  29. cartography/models/aws/glue/__init__.py +0 -0
  30. cartography/models/aws/glue/connection.py +51 -0
  31. cartography/models/aws/identitycenter/awspermissionset.py +44 -0
  32. cartography/models/aws/kms/__init__.py +0 -0
  33. cartography/models/aws/kms/aliases.py +86 -0
  34. cartography/models/aws/kms/grants.py +65 -0
  35. cartography/models/aws/kms/keys.py +88 -0
  36. cartography/models/aws/lambda_function/__init__.py +0 -0
  37. cartography/models/aws/lambda_function/alias.py +74 -0
  38. cartography/models/aws/lambda_function/event_source_mapping.py +88 -0
  39. cartography/models/aws/lambda_function/lambda_function.py +89 -0
  40. cartography/models/aws/lambda_function/layer.py +72 -0
  41. cartography/models/aws/rds/__init__.py +0 -0
  42. cartography/models/aws/rds/cluster.py +89 -0
  43. cartography/models/aws/rds/instance.py +154 -0
  44. cartography/models/aws/rds/snapshot.py +108 -0
  45. cartography/models/aws/rds/subnet_group.py +101 -0
  46. cartography/models/aws/route53/__init__.py +0 -0
  47. cartography/models/aws/route53/dnsrecord.py +214 -0
  48. cartography/models/aws/route53/nameserver.py +63 -0
  49. cartography/models/aws/route53/subzone.py +40 -0
  50. cartography/models/aws/route53/zone.py +47 -0
  51. cartography/models/aws/secretsmanager/secret.py +106 -0
  52. cartography/models/entra/group.py +26 -0
  53. cartography/models/entra/user.py +6 -0
  54. cartography/models/gcp/compute/__init__.py +0 -0
  55. cartography/models/gcp/compute/vpc.py +50 -0
  56. cartography/util.py +8 -1
  57. {cartography-0.108.0rc2.dist-info → cartography-0.109.0.dist-info}/METADATA +2 -2
  58. {cartography-0.108.0rc2.dist-info → cartography-0.109.0.dist-info}/RECORD +62 -38
  59. cartography/data/jobs/cleanup/aws_dns_cleanup.json +0 -65
  60. cartography/data/jobs/cleanup/aws_import_identity_center_cleanup.json +0 -16
  61. cartography/data/jobs/cleanup/aws_import_lambda_cleanup.json +0 -50
  62. cartography/data/jobs/cleanup/aws_import_rds_clusters_cleanup.json +0 -23
  63. cartography/data/jobs/cleanup/aws_import_rds_instances_cleanup.json +0 -47
  64. cartography/data/jobs/cleanup/aws_import_rds_snapshots_cleanup.json +0 -23
  65. cartography/data/jobs/cleanup/aws_import_secrets_cleanup.json +0 -8
  66. cartography/data/jobs/cleanup/aws_kms_details.json +0 -10
  67. {cartography-0.108.0rc2.dist-info → cartography-0.109.0.dist-info}/WHEEL +0 -0
  68. {cartography-0.108.0rc2.dist-info → cartography-0.109.0.dist-info}/entry_points.txt +0 -0
  69. {cartography-0.108.0rc2.dist-info → cartography-0.109.0.dist-info}/licenses/LICENSE +0 -0
  70. {cartography-0.108.0rc2.dist-info → cartography-0.109.0.dist-info}/top_level.txt +0 -0
cartography/_version.py CHANGED
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '0.108.0rc2'
21
- __version_tuple__ = version_tuple = (0, 108, 0, 'rc2')
20
+ __version__ = version = '0.109.0'
21
+ __version_tuple__ = version_tuple = (0, 109, 0)
cartography/cli.py CHANGED
@@ -700,6 +700,15 @@ class CLI:
700
700
  "Required if you are using the Trivy module. Ignored otherwise."
701
701
  ),
702
702
  )
703
+ parser.add_argument(
704
+ "--trivy-results-dir",
705
+ type=str,
706
+ default=None,
707
+ help=(
708
+ "Path to a directory containing Trivy JSON results on disk. "
709
+ "Required if you are using the Trivy module with local results."
710
+ ),
711
+ )
703
712
  parser.add_argument(
704
713
  "--scaleway-org",
705
714
  type=str,
@@ -1089,6 +1098,9 @@ class CLI:
1089
1098
  if config.trivy_s3_prefix:
1090
1099
  logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
1091
1100
 
1101
+ if config.trivy_results_dir:
1102
+ logger.debug(f"Trivy results dir: {config.trivy_results_dir}")
1103
+
1092
1104
  # Scaleway config
1093
1105
  if config.scaleway_secret_key_env_var:
1094
1106
  logger.debug(
@@ -1118,6 +1130,8 @@ class CLI:
1118
1130
  config.sentinelone_api_token = os.environ.get(
1119
1131
  config.sentinelone_api_token_env_var
1120
1132
  )
1133
+ else:
1134
+ config.sentinelone_api_token = None
1121
1135
 
1122
1136
  # Run cartography
1123
1137
  try:
cartography/config.py CHANGED
@@ -152,6 +152,8 @@ class Config:
152
152
  :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
153
153
  :type trivy_s3_prefix: str
154
154
  :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
155
+ :type trivy_results_dir: str
156
+ :param trivy_results_dir: Local directory containing Trivy scan results. Optional.
155
157
  :type scaleway_access_key: str
156
158
  :param scaleway_access_key: Scaleway access key. Optional.
157
159
  :type scaleway_secret_key: str
@@ -243,6 +245,7 @@ class Config:
243
245
  airbyte_api_url=None,
244
246
  trivy_s3_bucket=None,
245
247
  trivy_s3_prefix=None,
248
+ trivy_results_dir=None,
246
249
  scaleway_access_key=None,
247
250
  scaleway_secret_key=None,
248
251
  scaleway_org=None,
@@ -327,6 +330,7 @@ class Config:
327
330
  self.airbyte_api_url = airbyte_api_url
328
331
  self.trivy_s3_bucket = trivy_s3_bucket
329
332
  self.trivy_s3_prefix = trivy_s3_prefix
333
+ self.trivy_results_dir = trivy_results_dir
330
334
  self.scaleway_access_key = scaleway_access_key
331
335
  self.scaleway_secret_key = scaleway_secret_key
332
336
  self.scaleway_org = scaleway_org
@@ -29,14 +29,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
29
29
  CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.lastupdated);
30
30
  CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.id);
31
31
  CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.lastupdated);
32
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.id);
33
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.lastupdated);
34
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.id);
35
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.lastupdated);
36
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.id);
37
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.lastupdated);
38
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.id);
39
- CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.lastupdated);
40
32
  CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.id);
41
33
  CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.lastupdated);
42
34
  CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.id);
@@ -158,13 +150,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
158
150
  CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
159
151
  CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
160
152
  CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
161
- CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);
162
- CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.arn);
163
- CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.lastupdated);
164
- CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.id);
165
- CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.lastupdated);
166
- CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.id);
167
- CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
168
153
  CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
169
154
  CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
170
155
  CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);
@@ -259,8 +244,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.id);
259
244
  CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.name);
260
245
  CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.arn);
261
246
  CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.lastupdated);
262
- CREATE INDEX IF NOT EXISTS FOR (n:SecretsManagerSecret) ON (n.id);
263
- CREATE INDEX IF NOT EXISTS FOR (n:SecretsManagerSecret) ON (n.lastupdated);
264
247
  CREATE INDEX IF NOT EXISTS FOR (n:SecurityHub) ON (n.id);
265
248
  CREATE INDEX IF NOT EXISTS FOR (n:SecurityHub) ON (n.lastupdated);
266
249
  CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.id);
@@ -1,17 +1,5 @@
1
1
  {
2
2
  "statements": [
3
- {
4
- "query": "MATCH (n:GCPVpc) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
5
- "iterative": true,
6
- "iterationsize": 100,
7
- "__comment__": "Delete GCP VPCs that no longer exist and detach them from all previously connected nodes."
8
- },
9
- {
10
- "query": "MATCH (:GCPVpc)<-[r:RESOURCE]-(:GCPProject) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
11
- "iterative": true,
12
- "iterationsize": 100,
13
- "__comment__": "Remove GCP VPC-to-Project relationships that are out of date."
14
- },
15
3
  {
16
4
  "query": "MATCH (:GCPInstance)-[r:MEMBER_OF_GCP_VPC]->(:GCPVpc) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
17
5
  "iterative": true,
@@ -223,6 +223,13 @@ def transform_assume_role_events_to_role_assumptions(
223
223
 
224
224
  cloudtrail_event = json.loads(event["CloudTrailEvent"])
225
225
 
226
+ # Skip events with null requestParameters since we can't extract roleArn
227
+ if not cloudtrail_event.get("requestParameters"):
228
+ logger.debug(
229
+ f"Skipping CloudTrail AssumeRole event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
230
+ )
231
+ continue
232
+
226
233
  if cloudtrail_event.get("userIdentity", {}).get("arn"):
227
234
  source_principal = cloudtrail_event["userIdentity"]["arn"]
228
235
  destination_principal = cloudtrail_event["requestParameters"]["roleArn"]
@@ -298,6 +305,13 @@ def transform_saml_role_events_to_role_assumptions(
298
305
 
299
306
  cloudtrail_event = json.loads(event["CloudTrailEvent"])
300
307
 
308
+ # Skip events with null requestParameters since we can't extract roleArn
309
+ if not cloudtrail_event.get("requestParameters"):
310
+ logger.debug(
311
+ f"Skipping CloudTrail AssumeRoleWithSAML event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
312
+ )
313
+ continue
314
+
301
315
  response_elements = cloudtrail_event.get("responseElements", {})
302
316
  assumed_role_user = response_elements.get("assumedRoleUser", {})
303
317
 
@@ -370,6 +384,13 @@ def transform_web_identity_role_events_to_role_assumptions(
370
384
 
371
385
  cloudtrail_event = json.loads(event["CloudTrailEvent"])
372
386
 
387
+ # Skip events with null requestParameters since we can't extract roleArn
388
+ if not cloudtrail_event.get("requestParameters"):
389
+ logger.debug(
390
+ f"Skipping CloudTrail AssumeRoleWithWebIdentity event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
391
+ )
392
+ continue
393
+
373
394
  user_identity = cloudtrail_event.get("userIdentity", {})
374
395
 
375
396
  if user_identity.get("type") == "WebIdentityUser" and user_identity.get(
@@ -381,13 +402,15 @@ def transform_web_identity_role_events_to_role_assumptions(
381
402
 
382
403
  # Only process GitHub Actions events
383
404
  if "token.actions.githubusercontent.com" in identity_provider:
384
- # GitHub repo fullname is directly in userName (e.g., "sublimagesec/sublimage")
385
- github_repo = user_identity.get("userName", "")
386
- if not github_repo:
405
+ # Extract GitHub repo fullname from userName format: "repo:{organization}/{repository}:{context}"
406
+ user_name = user_identity.get("userName", "")
407
+ if not user_name:
387
408
  logger.debug(
388
409
  f"Missing userName in GitHub WebIdentity event: {event.get('EventId', 'unknown')}"
389
410
  )
390
411
  continue
412
+
413
+ github_repo = _extract_github_repo_from_username(user_name)
391
414
  key = (github_repo, destination_principal)
392
415
 
393
416
  if key in github_aggregated:
@@ -572,6 +595,37 @@ def _convert_assumed_role_arn_to_role_arn(assumed_role_arn: str) -> str:
572
595
  return assumed_role_arn
573
596
 
574
597
 
598
+ def _extract_github_repo_from_username(user_name: str) -> str:
599
+ """
600
+ Extract GitHub repository fullname from CloudTrail userName field.
601
+
602
+ GitHub Actions CloudTrail events have userName in the format:
603
+ "repo:{organization}/{repository}:{context}"
604
+ """
605
+ if not user_name:
606
+ return ""
607
+
608
+ parts = user_name.split(":")
609
+
610
+ # Need at least 3 parts: ["repo", "{organization}/{repository}", "{context}"]
611
+ if len(parts) < 3 or parts[0] != "repo":
612
+ return ""
613
+
614
+ # Extract "{organization}/{repository}"
615
+ repo_fullname = parts[1]
616
+
617
+ # Validate it looks like "{organization}/{repository}" format
618
+ if repo_fullname.count("/") != 1:
619
+ return ""
620
+
621
+ # Ensure both organization and repo exist
622
+ owner, repo = repo_fullname.split("/")
623
+ if not owner or not repo:
624
+ return ""
625
+
626
+ return repo_fullname
627
+
628
+
575
629
  @timeit
576
630
  def cleanup(
577
631
  neo4j_session: neo4j.Session, current_aws_account_id: str, update_tag: int
@@ -6,9 +6,12 @@ from typing import List
6
6
  import boto3
7
7
  import neo4j
8
8
 
9
+ from cartography.client.core.tx import load
10
+ from cartography.graph.job import GraphJob
11
+ from cartography.models.aws.ecr.image import ECRImageSchema
12
+ from cartography.models.aws.ecr.repository import ECRRepositorySchema
13
+ from cartography.models.aws.ecr.repository_image import ECRRepositoryImageSchema
9
14
  from cartography.util import aws_handle_regions
10
- from cartography.util import batch
11
- from cartography.util import run_cleanup_job
12
15
  from cartography.util import timeit
13
16
  from cartography.util import to_asynchronous
14
17
  from cartography.util import to_synchronous
@@ -74,33 +77,17 @@ def load_ecr_repositories(
74
77
  current_aws_account_id: str,
75
78
  aws_update_tag: int,
76
79
  ) -> None:
77
- query = """
78
- UNWIND $Repositories as ecr_repo
79
- MERGE (repo:ECRRepository{id: ecr_repo.repositoryArn})
80
- ON CREATE SET repo.firstseen = timestamp(),
81
- repo.arn = ecr_repo.repositoryArn,
82
- repo.name = ecr_repo.repositoryName,
83
- repo.region = $Region,
84
- repo.created_at = ecr_repo.createdAt
85
- SET repo.lastupdated = $aws_update_tag,
86
- repo.uri = ecr_repo.repositoryUri
87
- WITH repo
88
-
89
- MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
90
- MERGE (owner)-[r:RESOURCE]->(repo)
91
- ON CREATE SET r.firstseen = timestamp()
92
- SET r.lastupdated = $aws_update_tag
93
- """
94
80
  logger.info(
95
81
  f"Loading {len(repos)} ECR repositories for region {region} into graph.",
96
82
  )
97
- neo4j_session.run(
98
- query,
99
- Repositories=repos,
83
+ load(
84
+ neo4j_session,
85
+ ECRRepositorySchema(),
86
+ repos,
87
+ lastupdated=aws_update_tag,
100
88
  Region=region,
101
- aws_update_tag=aws_update_tag,
102
- AWS_ACCOUNT_ID=current_aws_account_id,
103
- ).consume() # See issue #440
89
+ AWS_ID=current_aws_account_id,
90
+ )
104
91
 
105
92
 
106
93
  @timeit
@@ -114,8 +101,13 @@ def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
114
101
  for repo_uri in sorted(repo_data.keys()):
115
102
  repo_images = repo_data[repo_uri]
116
103
  for img in repo_images:
117
- if "imageDigest" in img and img["imageDigest"]:
104
+ digest = img.get("imageDigest")
105
+ if digest:
106
+ tag = img.get("imageTag")
107
+ uri = repo_uri + (f":{tag}" if tag else "")
118
108
  img["repo_uri"] = repo_uri
109
+ img["uri"] = uri
110
+ img["id"] = uri
119
111
  repo_images_list.append(img)
120
112
  else:
121
113
  logger.warning(
@@ -127,74 +119,51 @@ def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
127
119
  return repo_images_list
128
120
 
129
121
 
130
- def _load_ecr_repo_img_tx(
131
- tx: neo4j.Transaction,
132
- repo_images_list: List[Dict],
133
- aws_update_tag: int,
134
- region: str,
135
- ) -> None:
136
- query = """
137
- UNWIND $RepoList as repo_img
138
- MERGE (ri:ECRRepositoryImage{id: repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')})
139
- ON CREATE SET ri.firstseen = timestamp()
140
- SET ri.lastupdated = $aws_update_tag,
141
- ri.tag = repo_img.imageTag,
142
- ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, ''),
143
- ri.image_size_bytes = repo_img.imageSizeInBytes,
144
- ri.image_pushed_at = repo_img.imagePushedAt,
145
- ri.image_manifest_media_type = repo_img.imageManifestMediaType,
146
- ri.artifact_media_type = repo_img.artifactMediaType,
147
- ri.last_recorded_pull_time = repo_img.lastRecordedPullTime
148
- WITH ri, repo_img
149
-
150
- MERGE (img:ECRImage{id: repo_img.imageDigest})
151
- ON CREATE SET img.firstseen = timestamp(),
152
- img.digest = repo_img.imageDigest
153
- SET img.lastupdated = $aws_update_tag,
154
- img.region = $Region
155
- WITH ri, img, repo_img
156
-
157
- MERGE (ri)-[r1:IMAGE]->(img)
158
- ON CREATE SET r1.firstseen = timestamp()
159
- SET r1.lastupdated = $aws_update_tag
160
- WITH ri, repo_img
161
-
162
- MATCH (repo:ECRRepository{uri: repo_img.repo_uri})
163
- MERGE (repo)-[r2:REPO_IMAGE]->(ri)
164
- ON CREATE SET r2.firstseen = timestamp()
165
- SET r2.lastupdated = $aws_update_tag
166
- """
167
- tx.run(
168
- query,
169
- RepoList=repo_images_list,
170
- Region=region,
171
- aws_update_tag=aws_update_tag,
172
- )
173
-
174
-
175
122
  @timeit
176
123
  def load_ecr_repository_images(
177
124
  neo4j_session: neo4j.Session,
178
125
  repo_images_list: List[Dict],
179
126
  region: str,
127
+ current_aws_account_id: str,
180
128
  aws_update_tag: int,
181
129
  ) -> None:
182
130
  logger.info(
183
131
  f"Loading {len(repo_images_list)} ECR repository images in {region} into graph.",
184
132
  )
185
- for repo_image_batch in batch(repo_images_list, size=10000):
186
- neo4j_session.write_transaction(
187
- _load_ecr_repo_img_tx,
188
- repo_image_batch,
189
- aws_update_tag,
190
- region,
191
- )
133
+ image_digests = {img["imageDigest"] for img in repo_images_list}
134
+ ecr_images = [{"imageDigest": d} for d in image_digests]
135
+
136
+ load(
137
+ neo4j_session,
138
+ ECRImageSchema(),
139
+ ecr_images,
140
+ lastupdated=aws_update_tag,
141
+ Region=region,
142
+ AWS_ID=current_aws_account_id,
143
+ )
144
+
145
+ load(
146
+ neo4j_session,
147
+ ECRRepositoryImageSchema(),
148
+ repo_images_list,
149
+ lastupdated=aws_update_tag,
150
+ Region=region,
151
+ AWS_ID=current_aws_account_id,
152
+ )
192
153
 
193
154
 
194
155
  @timeit
195
156
  def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
196
157
  logger.debug("Running ECR cleanup job.")
197
- run_cleanup_job("aws_import_ecr_cleanup.json", neo4j_session, common_job_parameters)
158
+ GraphJob.from_node_schema(ECRRepositorySchema(), common_job_parameters).run(
159
+ neo4j_session
160
+ )
161
+ GraphJob.from_node_schema(ECRRepositoryImageSchema(), common_job_parameters).run(
162
+ neo4j_session
163
+ )
164
+ GraphJob.from_node_schema(ECRImageSchema(), common_job_parameters).run(
165
+ neo4j_session
166
+ )
198
167
 
199
168
 
200
169
  def _get_image_data(
@@ -251,5 +220,11 @@ def sync(
251
220
  update_tag,
252
221
  )
253
222
  repo_images_list = transform_ecr_repository_images(image_data)
254
- load_ecr_repository_images(neo4j_session, repo_images_list, region, update_tag)
223
+ load_ecr_repository_images(
224
+ neo4j_session,
225
+ repo_images_list,
226
+ region,
227
+ current_aws_account_id,
228
+ update_tag,
229
+ )
255
230
  cleanup(neo4j_session, common_job_parameters)
@@ -0,0 +1,91 @@
1
+ import logging
2
+ from typing import Any
3
+ from typing import Dict
4
+ from typing import List
5
+
6
+ import boto3
7
+ import neo4j
8
+
9
+ from cartography.client.core.tx import load
10
+ from cartography.graph.job import GraphJob
11
+ from cartography.intel.aws.ec2.util import get_botocore_config
12
+ from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
13
+ from cartography.util import aws_handle_regions
14
+ from cartography.util import timeit
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @timeit
20
+ @aws_handle_regions
21
+ def get_eventbridge_rules(
22
+ boto3_session: boto3.Session, region: str
23
+ ) -> List[Dict[str, Any]]:
24
+ client = boto3_session.client(
25
+ "events", region_name=region, config=get_botocore_config()
26
+ )
27
+ paginator = client.get_paginator("list_rules")
28
+ rules = []
29
+
30
+ for page in paginator.paginate():
31
+ rules.extend(page.get("Rules", []))
32
+
33
+ return rules
34
+
35
+
36
+ @timeit
37
+ def load_eventbridge_rules(
38
+ neo4j_session: neo4j.Session,
39
+ data: List[Dict[str, Any]],
40
+ region: str,
41
+ current_aws_account_id: str,
42
+ aws_update_tag: int,
43
+ ) -> None:
44
+ logger.info(
45
+ f"Loading EventBridge {len(data)} rules for region '{region}' into graph.",
46
+ )
47
+ load(
48
+ neo4j_session,
49
+ EventBridgeRuleSchema(),
50
+ data,
51
+ lastupdated=aws_update_tag,
52
+ Region=region,
53
+ AWS_ID=current_aws_account_id,
54
+ )
55
+
56
+
57
+ @timeit
58
+ def cleanup(
59
+ neo4j_session: neo4j.Session,
60
+ common_job_parameters: Dict[str, Any],
61
+ ) -> None:
62
+ logger.debug("Running EventBridge cleanup job.")
63
+ GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
64
+ neo4j_session
65
+ )
66
+
67
+
68
+ @timeit
69
+ def sync(
70
+ neo4j_session: neo4j.Session,
71
+ boto3_session: boto3.session.Session,
72
+ regions: List[str],
73
+ current_aws_account_id: str,
74
+ update_tag: int,
75
+ common_job_parameters: Dict[str, Any],
76
+ ) -> None:
77
+ for region in regions:
78
+ logger.info(
79
+ f"Syncing EventBridge for region '{region}' in account '{current_aws_account_id}'.",
80
+ )
81
+
82
+ rules = get_eventbridge_rules(boto3_session, region)
83
+ load_eventbridge_rules(
84
+ neo4j_session,
85
+ rules,
86
+ region,
87
+ current_aws_account_id,
88
+ update_tag,
89
+ )
90
+
91
+ cleanup(neo4j_session, common_job_parameters)
@@ -0,0 +1,117 @@
1
+ import logging
2
+ from typing import Any
3
+ from typing import Dict
4
+ from typing import List
5
+
6
+ import boto3
7
+ import neo4j
8
+
9
+ from cartography.client.core.tx import load
10
+ from cartography.graph.job import GraphJob
11
+ from cartography.intel.aws.ec2.util import get_botocore_config
12
+ from cartography.models.aws.glue.connection import GlueConnectionSchema
13
+ from cartography.util import aws_handle_regions
14
+ from cartography.util import timeit
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @timeit
20
+ @aws_handle_regions
21
+ def get_glue_connections(
22
+ boto3_session: boto3.Session, region: str
23
+ ) -> List[Dict[str, Any]]:
24
+ client = boto3_session.client(
25
+ "glue", region_name=region, config=get_botocore_config()
26
+ )
27
+ paginator = client.get_paginator("get_connections")
28
+ connections = []
29
+ for page in paginator.paginate():
30
+ connections.extend(page.get("ConnectionList", []))
31
+
32
+ return connections
33
+
34
+
35
+ def transform_glue_connections(
36
+ connections: List[Dict[str, Any]], region: str
37
+ ) -> List[Dict[str, Any]]:
38
+ """
39
+ Transform Glue connection data for ingestion
40
+ """
41
+ transformed_connections = []
42
+ for connection in connections:
43
+ transformed_connection = {
44
+ "Name": connection["Name"],
45
+ "Description": connection.get("Description"),
46
+ "ConnectionType": connection.get("ConnectionType"),
47
+ "Status": connection.get("Status"),
48
+ "StatusReason": connection.get("StatusReason"),
49
+ "AuthenticationType": connection.get("AuthenticationConfiguration", {}).get(
50
+ "AuthenticationType"
51
+ ),
52
+ "SecretArn": connection.get("AuthenticationConfiguration", {}).get(
53
+ "SecretArn"
54
+ ),
55
+ "Region": region,
56
+ }
57
+ transformed_connections.append(transformed_connection)
58
+ return transformed_connections
59
+
60
+
61
+ @timeit
62
+ def load_glue_connections(
63
+ neo4j_session: neo4j.Session,
64
+ data: List[Dict[str, Any]],
65
+ region: str,
66
+ current_aws_account_id: str,
67
+ aws_update_tag: int,
68
+ ) -> None:
69
+ logger.info(
70
+ f"Loading Glue {len(data)} connections for region '{region}' into graph.",
71
+ )
72
+ load(
73
+ neo4j_session,
74
+ GlueConnectionSchema(),
75
+ data,
76
+ lastupdated=aws_update_tag,
77
+ Region=region,
78
+ AWS_ID=current_aws_account_id,
79
+ )
80
+
81
+
82
+ @timeit
83
+ def cleanup(
84
+ neo4j_session: neo4j.Session,
85
+ common_job_parameters: Dict[str, Any],
86
+ ) -> None:
87
+ logger.debug("Running Glue cleanup job.")
88
+ GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
89
+ neo4j_session
90
+ )
91
+
92
+
93
+ @timeit
94
+ def sync(
95
+ neo4j_session: neo4j.Session,
96
+ boto3_session: boto3.session.Session,
97
+ regions: List[str],
98
+ current_aws_account_id: str,
99
+ update_tag: int,
100
+ common_job_parameters: Dict[str, Any],
101
+ ) -> None:
102
+ for region in regions:
103
+ logger.info(
104
+ f"Syncing Glue for region '{region}' in account '{current_aws_account_id}'.",
105
+ )
106
+
107
+ connections = get_glue_connections(boto3_session, region)
108
+ transformed_connections = transform_glue_connections(connections, region)
109
+ load_glue_connections(
110
+ neo4j_session,
111
+ transformed_connections,
112
+ region,
113
+ current_aws_account_id,
114
+ update_tag,
115
+ )
116
+
117
+ cleanup(neo4j_session, common_job_parameters)