cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/__main__.py +1 -2
- cartography/_version.py +2 -2
- cartography/cli.py +376 -249
- cartography/client/core/tx.py +39 -18
- cartography/config.py +28 -0
- cartography/driftdetect/__main__.py +1 -2
- cartography/driftdetect/add_shortcut.py +10 -2
- cartography/driftdetect/cli.py +71 -75
- cartography/driftdetect/detect_deviations.py +7 -3
- cartography/driftdetect/get_states.py +20 -8
- cartography/driftdetect/model.py +5 -5
- cartography/driftdetect/serializers.py +8 -6
- cartography/driftdetect/storage.py +2 -2
- cartography/graph/cleanupbuilder.py +35 -15
- cartography/graph/job.py +46 -17
- cartography/graph/querybuilder.py +165 -80
- cartography/graph/statement.py +35 -26
- cartography/intel/analysis.py +4 -1
- cartography/intel/aws/__init__.py +114 -55
- cartography/intel/aws/apigateway.py +134 -63
- cartography/intel/aws/cloudtrail.py +127 -0
- cartography/intel/aws/cloudwatch.py +93 -0
- cartography/intel/aws/config.py +56 -20
- cartography/intel/aws/dynamodb.py +108 -40
- cartography/intel/aws/ec2/__init__.py +2 -2
- cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
- cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
- cartography/intel/aws/ec2/images.py +49 -20
- cartography/intel/aws/ec2/instances.py +234 -136
- cartography/intel/aws/ec2/internet_gateways.py +40 -11
- cartography/intel/aws/ec2/key_pairs.py +44 -20
- cartography/intel/aws/ec2/launch_templates.py +101 -59
- cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
- cartography/intel/aws/ec2/load_balancers.py +82 -42
- cartography/intel/aws/ec2/network_acls.py +89 -65
- cartography/intel/aws/ec2/network_interfaces.py +146 -87
- cartography/intel/aws/ec2/reserved_instances.py +45 -16
- cartography/intel/aws/ec2/route_tables.py +138 -98
- cartography/intel/aws/ec2/security_groups.py +71 -21
- cartography/intel/aws/ec2/snapshots.py +61 -22
- cartography/intel/aws/ec2/subnets.py +54 -18
- cartography/intel/aws/ec2/tgw.py +100 -34
- cartography/intel/aws/ec2/util.py +1 -1
- cartography/intel/aws/ec2/volumes.py +69 -41
- cartography/intel/aws/ec2/vpc.py +37 -12
- cartography/intel/aws/ec2/vpc_peerings.py +83 -24
- cartography/intel/aws/ecr.py +88 -32
- cartography/intel/aws/ecs.py +83 -47
- cartography/intel/aws/efs.py +93 -0
- cartography/intel/aws/eks.py +55 -29
- cartography/intel/aws/elasticache.py +42 -18
- cartography/intel/aws/elasticsearch.py +57 -20
- cartography/intel/aws/emr.py +61 -23
- cartography/intel/aws/iam.py +401 -145
- cartography/intel/aws/iam_instance_profiles.py +22 -22
- cartography/intel/aws/identitycenter.py +71 -37
- cartography/intel/aws/inspector.py +159 -89
- cartography/intel/aws/kms.py +92 -38
- cartography/intel/aws/lambda_function.py +103 -34
- cartography/intel/aws/organizations.py +30 -10
- cartography/intel/aws/permission_relationships.py +133 -51
- cartography/intel/aws/rds.py +249 -85
- cartography/intel/aws/redshift.py +107 -46
- cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
- cartography/intel/aws/resources.py +57 -46
- cartography/intel/aws/route53.py +108 -61
- cartography/intel/aws/s3.py +168 -83
- cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
- cartography/intel/aws/secretsmanager.py +24 -12
- cartography/intel/aws/securityhub.py +20 -9
- cartography/intel/aws/sns.py +166 -0
- cartography/intel/aws/sqs.py +60 -28
- cartography/intel/aws/ssm.py +70 -30
- cartography/intel/aws/util/arns.py +7 -7
- cartography/intel/aws/util/common.py +31 -4
- cartography/intel/azure/__init__.py +78 -19
- cartography/intel/azure/compute.py +101 -27
- cartography/intel/azure/cosmosdb.py +496 -170
- cartography/intel/azure/sql.py +296 -105
- cartography/intel/azure/storage.py +322 -113
- cartography/intel/azure/subscription.py +39 -23
- cartography/intel/azure/tenant.py +13 -4
- cartography/intel/azure/util/credentials.py +95 -55
- cartography/intel/bigfix/__init__.py +2 -2
- cartography/intel/bigfix/computers.py +93 -65
- cartography/intel/cloudflare/__init__.py +74 -0
- cartography/intel/cloudflare/accounts.py +57 -0
- cartography/intel/cloudflare/dnsrecords.py +64 -0
- cartography/intel/cloudflare/members.py +75 -0
- cartography/intel/cloudflare/roles.py +65 -0
- cartography/intel/cloudflare/zones.py +64 -0
- cartography/intel/create_indexes.py +3 -2
- cartography/intel/crowdstrike/__init__.py +11 -9
- cartography/intel/crowdstrike/endpoints.py +5 -1
- cartography/intel/crowdstrike/spotlight.py +8 -3
- cartography/intel/cve/__init__.py +46 -13
- cartography/intel/cve/feed.py +48 -12
- cartography/intel/digitalocean/__init__.py +22 -13
- cartography/intel/digitalocean/compute.py +75 -108
- cartography/intel/digitalocean/management.py +44 -80
- cartography/intel/digitalocean/platform.py +48 -43
- cartography/intel/dns.py +36 -10
- cartography/intel/duo/__init__.py +21 -16
- cartography/intel/duo/api_host.py +14 -9
- cartography/intel/duo/endpoints.py +50 -45
- cartography/intel/duo/groups.py +18 -14
- cartography/intel/duo/phones.py +37 -34
- cartography/intel/duo/tokens.py +26 -23
- cartography/intel/duo/users.py +54 -50
- cartography/intel/duo/web_authn_credentials.py +30 -25
- cartography/intel/entra/__init__.py +25 -7
- cartography/intel/entra/ou.py +112 -0
- cartography/intel/entra/users.py +69 -63
- cartography/intel/gcp/__init__.py +185 -49
- cartography/intel/gcp/compute.py +418 -231
- cartography/intel/gcp/crm.py +96 -43
- cartography/intel/gcp/dns.py +60 -19
- cartography/intel/gcp/gke.py +72 -38
- cartography/intel/gcp/iam.py +61 -41
- cartography/intel/gcp/storage.py +84 -55
- cartography/intel/github/__init__.py +13 -11
- cartography/intel/github/repos.py +270 -137
- cartography/intel/github/teams.py +170 -88
- cartography/intel/github/users.py +70 -39
- cartography/intel/github/util.py +36 -34
- cartography/intel/gsuite/__init__.py +47 -26
- cartography/intel/gsuite/api.py +73 -30
- cartography/intel/jamf/__init__.py +19 -1
- cartography/intel/jamf/computers.py +30 -7
- cartography/intel/jamf/util.py +7 -2
- cartography/intel/kandji/__init__.py +6 -3
- cartography/intel/kandji/devices.py +14 -8
- cartography/intel/kubernetes/namespaces.py +7 -4
- cartography/intel/kubernetes/pods.py +7 -4
- cartography/intel/kubernetes/services.py +8 -4
- cartography/intel/lastpass/__init__.py +2 -2
- cartography/intel/lastpass/users.py +23 -12
- cartography/intel/oci/__init__.py +44 -11
- cartography/intel/oci/iam.py +134 -38
- cartography/intel/oci/organizations.py +13 -6
- cartography/intel/oci/utils.py +43 -20
- cartography/intel/okta/__init__.py +66 -15
- cartography/intel/okta/applications.py +42 -20
- cartography/intel/okta/awssaml.py +93 -33
- cartography/intel/okta/factors.py +16 -4
- cartography/intel/okta/groups.py +56 -29
- cartography/intel/okta/organization.py +5 -1
- cartography/intel/okta/origins.py +6 -2
- cartography/intel/okta/roles.py +15 -5
- cartography/intel/okta/users.py +20 -8
- cartography/intel/okta/utils.py +6 -4
- cartography/intel/openai/__init__.py +86 -0
- cartography/intel/openai/adminapikeys.py +90 -0
- cartography/intel/openai/apikeys.py +96 -0
- cartography/intel/openai/projects.py +94 -0
- cartography/intel/openai/serviceaccounts.py +82 -0
- cartography/intel/openai/users.py +78 -0
- cartography/intel/openai/util.py +29 -0
- cartography/intel/pagerduty/__init__.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +18 -6
- cartography/intel/pagerduty/schedules.py +12 -4
- cartography/intel/pagerduty/services.py +11 -4
- cartography/intel/pagerduty/teams.py +8 -3
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/semgrep/__init__.py +24 -6
- cartography/intel/semgrep/dependencies.py +50 -28
- cartography/intel/semgrep/deployment.py +3 -1
- cartography/intel/semgrep/findings.py +42 -18
- cartography/intel/snipeit/__init__.py +17 -3
- cartography/intel/snipeit/asset.py +12 -6
- cartography/intel/snipeit/user.py +8 -5
- cartography/intel/snipeit/util.py +9 -4
- cartography/intel/tailscale/__init__.py +77 -0
- cartography/intel/tailscale/acls.py +146 -0
- cartography/intel/tailscale/devices.py +127 -0
- cartography/intel/tailscale/postureintegrations.py +81 -0
- cartography/intel/tailscale/tailnets.py +76 -0
- cartography/intel/tailscale/users.py +80 -0
- cartography/intel/tailscale/utils.py +132 -0
- cartography/models/aws/apigateway.py +21 -17
- cartography/models/aws/apigatewaycertificate.py +28 -22
- cartography/models/aws/apigatewayresource.py +28 -20
- cartography/models/aws/apigatewaystage.py +33 -25
- cartography/models/aws/cloudtrail/__init__.py +0 -0
- cartography/models/aws/cloudtrail/trail.py +61 -0
- cartography/models/aws/cloudwatch/__init__.py +0 -0
- cartography/models/aws/cloudwatch/loggroup.py +52 -0
- cartography/models/aws/dynamodb/gsi.py +30 -22
- cartography/models/aws/dynamodb/tables.py +25 -17
- cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
- cartography/models/aws/ec2/images.py +36 -34
- cartography/models/aws/ec2/instances.py +51 -45
- cartography/models/aws/ec2/keypair.py +21 -16
- cartography/models/aws/ec2/keypair_instance.py +28 -21
- cartography/models/aws/ec2/launch_configurations.py +30 -26
- cartography/models/aws/ec2/launch_template_versions.py +48 -38
- cartography/models/aws/ec2/launch_templates.py +21 -17
- cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
- cartography/models/aws/ec2/load_balancers.py +47 -37
- cartography/models/aws/ec2/network_acl_rules.py +38 -30
- cartography/models/aws/ec2/network_acls.py +38 -29
- cartography/models/aws/ec2/networkinterface_instance.py +52 -39
- cartography/models/aws/ec2/networkinterfaces.py +53 -37
- cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
- cartography/models/aws/ec2/reservations.py +18 -14
- cartography/models/aws/ec2/route_table_associations.py +44 -34
- cartography/models/aws/ec2/route_tables.py +50 -43
- cartography/models/aws/ec2/routes.py +45 -37
- cartography/models/aws/ec2/securitygroup_instance.py +29 -20
- cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
- cartography/models/aws/ec2/subnet_instance.py +24 -19
- cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
- cartography/models/aws/ec2/volumes.py +47 -40
- cartography/models/aws/efs/__init__.py +0 -0
- cartography/models/aws/efs/mount_target.py +52 -0
- cartography/models/aws/eks/clusters.py +23 -21
- cartography/models/aws/emr.py +32 -30
- cartography/models/aws/iam/instanceprofile.py +33 -24
- cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
- cartography/models/aws/identitycenter/awspermissionset.py +37 -29
- cartography/models/aws/identitycenter/awsssouser.py +23 -21
- cartography/models/aws/inspector/findings.py +77 -65
- cartography/models/aws/inspector/packages.py +35 -29
- cartography/models/aws/s3/__init__.py +0 -0
- cartography/models/aws/s3/account_public_access_block.py +51 -0
- cartography/models/aws/sns/__init__.py +0 -0
- cartography/models/aws/sns/topic.py +50 -0
- cartography/models/aws/ssm/instance_information.py +51 -39
- cartography/models/aws/ssm/instance_patch.py +32 -26
- cartography/models/bigfix/bigfix_computer.py +42 -38
- cartography/models/bigfix/bigfix_root.py +3 -3
- cartography/models/cloudflare/__init__.py +0 -0
- cartography/models/cloudflare/account.py +25 -0
- cartography/models/cloudflare/dnsrecord.py +55 -0
- cartography/models/cloudflare/member.py +82 -0
- cartography/models/cloudflare/role.py +44 -0
- cartography/models/cloudflare/zone.py +59 -0
- cartography/models/core/common.py +12 -10
- cartography/models/core/nodes.py +5 -2
- cartography/models/core/relationships.py +14 -6
- cartography/models/crowdstrike/hosts.py +37 -35
- cartography/models/cve/cve.py +34 -32
- cartography/models/cve/cve_feed.py +6 -6
- cartography/models/digitalocean/__init__.py +0 -0
- cartography/models/digitalocean/account.py +21 -0
- cartography/models/digitalocean/droplet.py +56 -0
- cartography/models/digitalocean/project.py +48 -0
- cartography/models/duo/api_host.py +3 -3
- cartography/models/duo/endpoint.py +43 -41
- cartography/models/duo/group.py +14 -14
- cartography/models/duo/phone.py +27 -27
- cartography/models/duo/token.py +16 -16
- cartography/models/duo/user.py +46 -44
- cartography/models/duo/web_authn_credential.py +27 -19
- cartography/models/entra/ou.py +48 -0
- cartography/models/entra/tenant.py +24 -18
- cartography/models/entra/user.py +64 -48
- cartography/models/gcp/iam.py +23 -23
- cartography/models/github/orgs.py +5 -4
- cartography/models/github/teams.py +37 -31
- cartography/models/github/users.py +34 -23
- cartography/models/kandji/device.py +22 -16
- cartography/models/kandji/tenant.py +6 -4
- cartography/models/lastpass/tenant.py +3 -3
- cartography/models/lastpass/user.py +32 -28
- cartography/models/openai/__init__.py +0 -0
- cartography/models/openai/adminapikey.py +90 -0
- cartography/models/openai/apikey.py +84 -0
- cartography/models/openai/organization.py +17 -0
- cartography/models/openai/project.py +70 -0
- cartography/models/openai/serviceaccount.py +50 -0
- cartography/models/openai/user.py +49 -0
- cartography/models/semgrep/dependencies.py +36 -24
- cartography/models/semgrep/deployment.py +5 -5
- cartography/models/semgrep/findings.py +58 -42
- cartography/models/semgrep/locations.py +27 -21
- cartography/models/snipeit/asset.py +30 -21
- cartography/models/snipeit/tenant.py +6 -4
- cartography/models/snipeit/user.py +19 -12
- cartography/models/tailscale/__init__.py +0 -0
- cartography/models/tailscale/device.py +95 -0
- cartography/models/tailscale/group.py +86 -0
- cartography/models/tailscale/postureintegration.py +58 -0
- cartography/models/tailscale/tag.py +102 -0
- cartography/models/tailscale/tailnet.py +29 -0
- cartography/models/tailscale/user.py +52 -0
- cartography/stats.py +3 -3
- cartography/sync.py +113 -31
- cartography/util.py +84 -62
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/METADATA +8 -15
- cartography-0.103.0.dist-info/RECORD +442 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/WHEEL +1 -1
- cartography-0.102.0rc2.dist-info/RECORD +0 -381
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/entry_points.txt +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/top_level.txt +0 -0
cartography/intel/aws/s3.py
CHANGED
|
@@ -30,16 +30,20 @@ stat_handler = get_stats_client(__name__)
|
|
|
30
30
|
|
|
31
31
|
@timeit
|
|
32
32
|
def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
|
|
33
|
-
client = boto3_session.client(
|
|
33
|
+
client = boto3_session.client("s3")
|
|
34
34
|
# NOTE no paginator available for this operation
|
|
35
35
|
buckets = client.list_buckets()
|
|
36
|
-
for bucket in buckets[
|
|
36
|
+
for bucket in buckets["Buckets"]:
|
|
37
37
|
try:
|
|
38
|
-
bucket[
|
|
38
|
+
bucket["Region"] = client.get_bucket_location(Bucket=bucket["Name"])[
|
|
39
|
+
"LocationConstraint"
|
|
40
|
+
]
|
|
39
41
|
except ClientError as e:
|
|
40
42
|
if _is_common_exception(e, bucket):
|
|
41
|
-
bucket[
|
|
42
|
-
logger.warning(
|
|
43
|
+
bucket["Region"] = None
|
|
44
|
+
logger.warning(
|
|
45
|
+
"skipping bucket='{}' due to exception.".format(bucket["Name"]),
|
|
46
|
+
)
|
|
43
47
|
continue
|
|
44
48
|
else:
|
|
45
49
|
raise
|
|
@@ -48,8 +52,8 @@ def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
|
|
|
48
52
|
|
|
49
53
|
@timeit
|
|
50
54
|
def get_s3_bucket_details(
|
|
51
|
-
|
|
52
|
-
|
|
55
|
+
boto3_session: boto3.session.Session,
|
|
56
|
+
bucket_data: Dict,
|
|
53
57
|
) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict], None, None]:
|
|
54
58
|
"""
|
|
55
59
|
Iterates over all S3 buckets. Yields bucket name (string), S3 bucket policies (JSON), ACLs (JSON),
|
|
@@ -58,16 +62,23 @@ def get_s3_bucket_details(
|
|
|
58
62
|
# a local store for s3 clients so that we may re-use clients for an AWS region
|
|
59
63
|
s3_regional_clients: Dict[Any, Any] = {}
|
|
60
64
|
|
|
61
|
-
BucketDetail = Tuple[
|
|
65
|
+
BucketDetail = Tuple[
|
|
66
|
+
str,
|
|
67
|
+
Dict[str, Any],
|
|
68
|
+
Dict[str, Any],
|
|
69
|
+
Dict[str, Any],
|
|
70
|
+
Dict[str, Any],
|
|
71
|
+
Dict[str, Any],
|
|
72
|
+
]
|
|
62
73
|
|
|
63
74
|
async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail:
|
|
64
75
|
# Note: bucket['Region'] is sometimes None because
|
|
65
76
|
# client.get_bucket_location() does not return a location constraint for buckets
|
|
66
77
|
# in us-east-1 region
|
|
67
|
-
client = s3_regional_clients.get(bucket[
|
|
78
|
+
client = s3_regional_clients.get(bucket["Region"])
|
|
68
79
|
if not client:
|
|
69
|
-
client = boto3_session.client(
|
|
70
|
-
s3_regional_clients[bucket[
|
|
80
|
+
client = boto3_session.client("s3", bucket["Region"])
|
|
81
|
+
s3_regional_clients[bucket["Region"]] = client
|
|
71
82
|
(
|
|
72
83
|
acl,
|
|
73
84
|
policy,
|
|
@@ -81,9 +92,11 @@ def get_s3_bucket_details(
|
|
|
81
92
|
to_asynchronous(get_versioning, bucket, client),
|
|
82
93
|
to_asynchronous(get_public_access_block, bucket, client),
|
|
83
94
|
)
|
|
84
|
-
return bucket[
|
|
95
|
+
return bucket["Name"], acl, policy, encryption, versioning, public_access_block
|
|
85
96
|
|
|
86
|
-
bucket_details = to_synchronous(
|
|
97
|
+
bucket_details = to_synchronous(
|
|
98
|
+
*[_get_bucket_detail(bucket) for bucket in bucket_data["Buckets"]],
|
|
99
|
+
)
|
|
87
100
|
yield from bucket_details
|
|
88
101
|
|
|
89
102
|
|
|
@@ -94,7 +107,7 @@ def get_policy(bucket: Dict, client: botocore.client.BaseClient) -> Optional[Dic
|
|
|
94
107
|
"""
|
|
95
108
|
policy = None
|
|
96
109
|
try:
|
|
97
|
-
policy = client.get_bucket_policy(Bucket=bucket[
|
|
110
|
+
policy = client.get_bucket_policy(Bucket=bucket["Name"])
|
|
98
111
|
except ClientError as e:
|
|
99
112
|
if _is_common_exception(e, bucket):
|
|
100
113
|
pass
|
|
@@ -114,7 +127,7 @@ def get_acl(bucket: Dict, client: botocore.client.BaseClient) -> Optional[Dict]:
|
|
|
114
127
|
"""
|
|
115
128
|
acl = None
|
|
116
129
|
try:
|
|
117
|
-
acl = client.get_bucket_acl(Bucket=bucket[
|
|
130
|
+
acl = client.get_bucket_acl(Bucket=bucket["Name"])
|
|
118
131
|
except ClientError as e:
|
|
119
132
|
if _is_common_exception(e, bucket):
|
|
120
133
|
pass
|
|
@@ -134,7 +147,7 @@ def get_encryption(bucket: Dict, client: botocore.client.BaseClient) -> Optional
|
|
|
134
147
|
"""
|
|
135
148
|
encryption = None
|
|
136
149
|
try:
|
|
137
|
-
encryption = client.get_bucket_encryption(Bucket=bucket[
|
|
150
|
+
encryption = client.get_bucket_encryption(Bucket=bucket["Name"])
|
|
138
151
|
except ClientError as e:
|
|
139
152
|
if _is_common_exception(e, bucket):
|
|
140
153
|
pass
|
|
@@ -154,7 +167,7 @@ def get_versioning(bucket: Dict, client: botocore.client.BaseClient) -> Optional
|
|
|
154
167
|
"""
|
|
155
168
|
versioning = None
|
|
156
169
|
try:
|
|
157
|
-
versioning = client.get_bucket_versioning(Bucket=bucket[
|
|
170
|
+
versioning = client.get_bucket_versioning(Bucket=bucket["Name"])
|
|
158
171
|
except ClientError as e:
|
|
159
172
|
if _is_common_exception(e, bucket):
|
|
160
173
|
pass
|
|
@@ -168,13 +181,16 @@ def get_versioning(bucket: Dict, client: botocore.client.BaseClient) -> Optional
|
|
|
168
181
|
|
|
169
182
|
|
|
170
183
|
@timeit
|
|
171
|
-
def get_public_access_block(
|
|
184
|
+
def get_public_access_block(
|
|
185
|
+
bucket: Dict,
|
|
186
|
+
client: botocore.client.BaseClient,
|
|
187
|
+
) -> Optional[Dict]:
|
|
172
188
|
"""
|
|
173
189
|
Gets the S3 bucket public access block configuration.
|
|
174
190
|
"""
|
|
175
191
|
public_access_block = None
|
|
176
192
|
try:
|
|
177
|
-
public_access_block = client.get_public_access_block(Bucket=bucket[
|
|
193
|
+
public_access_block = client.get_public_access_block(Bucket=bucket["Name"])
|
|
178
194
|
except ClientError as e:
|
|
179
195
|
if _is_common_exception(e, bucket):
|
|
180
196
|
pass
|
|
@@ -207,26 +223,32 @@ def _is_common_exception(e: Exception, bucket: Dict) -> bool:
|
|
|
207
223
|
logger.warning(f"{error_msg} for {bucket['Name']} - EndpointConnectionError")
|
|
208
224
|
return True
|
|
209
225
|
elif "ServerSideEncryptionConfigurationNotFoundError" in e.args[0]:
|
|
210
|
-
logger.warning(
|
|
226
|
+
logger.warning(
|
|
227
|
+
f"{error_msg} for {bucket['Name']} - ServerSideEncryptionConfigurationNotFoundError",
|
|
228
|
+
)
|
|
211
229
|
return True
|
|
212
230
|
elif "InvalidToken" in e.args[0]:
|
|
213
231
|
logger.warning(f"{error_msg} for {bucket['Name']} - InvalidToken")
|
|
214
232
|
return True
|
|
215
233
|
elif "NoSuchPublicAccessBlockConfiguration" in e.args[0]:
|
|
216
|
-
logger.warning(
|
|
234
|
+
logger.warning(
|
|
235
|
+
f"{error_msg} for {bucket['Name']} - NoSuchPublicAccessBlockConfiguration",
|
|
236
|
+
)
|
|
217
237
|
return True
|
|
218
238
|
elif "IllegalLocationConstraintException" in e.args[0]:
|
|
219
|
-
logger.warning(
|
|
239
|
+
logger.warning(
|
|
240
|
+
f"{error_msg} for {bucket['Name']} - IllegalLocationConstraintException",
|
|
241
|
+
)
|
|
220
242
|
return True
|
|
221
243
|
return False
|
|
222
244
|
|
|
223
245
|
|
|
224
246
|
@timeit
|
|
225
247
|
def _load_s3_acls(
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
248
|
+
neo4j_session: neo4j.Session,
|
|
249
|
+
acls: List[Dict[str, Any]],
|
|
250
|
+
aws_account_id: str,
|
|
251
|
+
update_tag: int,
|
|
230
252
|
) -> None:
|
|
231
253
|
"""
|
|
232
254
|
Ingest S3 ACL into neo4j.
|
|
@@ -252,14 +274,18 @@ def _load_s3_acls(
|
|
|
252
274
|
# implement the acl permission
|
|
253
275
|
# https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions
|
|
254
276
|
run_analysis_job(
|
|
255
|
-
|
|
277
|
+
"aws_s3acl_analysis.json",
|
|
256
278
|
neo4j_session,
|
|
257
|
-
{
|
|
279
|
+
{"AWS_ID": aws_account_id},
|
|
258
280
|
)
|
|
259
281
|
|
|
260
282
|
|
|
261
283
|
@timeit
|
|
262
|
-
def _load_s3_policies(
|
|
284
|
+
def _load_s3_policies(
|
|
285
|
+
neo4j_session: neo4j.Session,
|
|
286
|
+
policies: List[Dict],
|
|
287
|
+
update_tag: int,
|
|
288
|
+
) -> None:
|
|
263
289
|
"""
|
|
264
290
|
Ingest S3 policy results into neo4j.
|
|
265
291
|
"""
|
|
@@ -281,7 +307,9 @@ def _load_s3_policies(neo4j_session: neo4j.Session, policies: List[Dict], update
|
|
|
281
307
|
|
|
282
308
|
@timeit
|
|
283
309
|
def _load_s3_policy_statements(
|
|
284
|
-
neo4j_session: neo4j.Session,
|
|
310
|
+
neo4j_session: neo4j.Session,
|
|
311
|
+
statements: List[Dict],
|
|
312
|
+
update_tag: int,
|
|
285
313
|
) -> None:
|
|
286
314
|
ingest_policy_statement = """
|
|
287
315
|
UNWIND $Statements as statement_data
|
|
@@ -311,7 +339,11 @@ def _load_s3_policy_statements(
|
|
|
311
339
|
|
|
312
340
|
|
|
313
341
|
@timeit
|
|
314
|
-
def _load_s3_encryption(
|
|
342
|
+
def _load_s3_encryption(
|
|
343
|
+
neo4j_session: neo4j.Session,
|
|
344
|
+
encryption_configs: List[Dict],
|
|
345
|
+
update_tag: int,
|
|
346
|
+
) -> None:
|
|
315
347
|
"""
|
|
316
348
|
Ingest S3 default encryption results into neo4j.
|
|
317
349
|
"""
|
|
@@ -333,7 +365,11 @@ def _load_s3_encryption(neo4j_session: neo4j.Session, encryption_configs: List[D
|
|
|
333
365
|
|
|
334
366
|
|
|
335
367
|
@timeit
|
|
336
|
-
def _load_s3_versioning(
|
|
368
|
+
def _load_s3_versioning(
|
|
369
|
+
neo4j_session: neo4j.Session,
|
|
370
|
+
versioning_configs: List[Dict],
|
|
371
|
+
update_tag: int,
|
|
372
|
+
) -> None:
|
|
337
373
|
"""
|
|
338
374
|
Ingest S3 versioning results into neo4j.
|
|
339
375
|
"""
|
|
@@ -400,7 +436,9 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No
|
|
|
400
436
|
|
|
401
437
|
@timeit
|
|
402
438
|
def load_s3_details(
|
|
403
|
-
neo4j_session: neo4j.Session,
|
|
439
|
+
neo4j_session: neo4j.Session,
|
|
440
|
+
s3_details_iter: Generator[Any, Any, Any],
|
|
441
|
+
aws_account_id: str,
|
|
404
442
|
update_tag: int,
|
|
405
443
|
) -> None:
|
|
406
444
|
"""
|
|
@@ -412,7 +450,14 @@ def load_s3_details(
|
|
|
412
450
|
encryption_configs: List[Dict] = []
|
|
413
451
|
versioning_configs: List[Dict] = []
|
|
414
452
|
public_access_block_configs: List[Dict] = []
|
|
415
|
-
for
|
|
453
|
+
for (
|
|
454
|
+
bucket,
|
|
455
|
+
acl,
|
|
456
|
+
policy,
|
|
457
|
+
encryption,
|
|
458
|
+
versioning,
|
|
459
|
+
public_access_block,
|
|
460
|
+
) in s3_details_iter:
|
|
416
461
|
parsed_acls = parse_acl(acl, bucket, aws_account_id)
|
|
417
462
|
if parsed_acls is not None:
|
|
418
463
|
acls.extend(parsed_acls)
|
|
@@ -428,15 +473,18 @@ def load_s3_details(
|
|
|
428
473
|
parsed_versioning = parse_versioning(bucket, versioning)
|
|
429
474
|
if parsed_versioning is not None:
|
|
430
475
|
versioning_configs.append(parsed_versioning)
|
|
431
|
-
parsed_public_access_block = parse_public_access_block(
|
|
476
|
+
parsed_public_access_block = parse_public_access_block(
|
|
477
|
+
bucket,
|
|
478
|
+
public_access_block,
|
|
479
|
+
)
|
|
432
480
|
if parsed_public_access_block is not None:
|
|
433
481
|
public_access_block_configs.append(parsed_public_access_block)
|
|
434
482
|
|
|
435
483
|
# cleanup existing policy properties set on S3 Buckets
|
|
436
484
|
run_cleanup_job(
|
|
437
|
-
|
|
485
|
+
"aws_s3_details.json",
|
|
438
486
|
neo4j_session,
|
|
439
|
-
{
|
|
487
|
+
{"UPDATE_TAG": update_tag, "AWS_ID": aws_account_id},
|
|
440
488
|
)
|
|
441
489
|
|
|
442
490
|
_load_s3_acls(neo4j_session, acls, aws_account_id, update_tag)
|
|
@@ -492,7 +540,7 @@ def parse_policy(bucket: str, policyDict: Optional[Dict]) -> Optional[Dict]:
|
|
|
492
540
|
if policyDict is None:
|
|
493
541
|
return None
|
|
494
542
|
# get just the policy element and convert to JSON because boto3 returns this as string
|
|
495
|
-
policy = Policy(json.loads(policyDict[
|
|
543
|
+
policy = Policy(json.loads(policyDict["Policy"]))
|
|
496
544
|
if policy.is_internet_accessible():
|
|
497
545
|
return {
|
|
498
546
|
"bucket": bucket,
|
|
@@ -512,7 +560,7 @@ def parse_policy_statements(bucket: str, policyDict: Policy) -> List[Dict]:
|
|
|
512
560
|
if policyDict is None:
|
|
513
561
|
return None
|
|
514
562
|
|
|
515
|
-
policy = json.loads(policyDict[
|
|
563
|
+
policy = json.loads(policyDict["Policy"])
|
|
516
564
|
statements = []
|
|
517
565
|
stmt_index = 1
|
|
518
566
|
for s in policy["Statement"]:
|
|
@@ -544,8 +592,12 @@ def parse_policy_statements(bucket: str, policyDict: Policy) -> List[Dict]:
|
|
|
544
592
|
|
|
545
593
|
|
|
546
594
|
@timeit
|
|
547
|
-
def parse_acl(
|
|
548
|
-
|
|
595
|
+
def parse_acl(
|
|
596
|
+
acl: Optional[Dict],
|
|
597
|
+
bucket: str,
|
|
598
|
+
aws_account_id: str,
|
|
599
|
+
) -> Optional[List[Dict]]:
|
|
600
|
+
"""Parses the AWS ACL object and returns a dict of the relevant data"""
|
|
549
601
|
# ACL JSON looks like
|
|
550
602
|
# ...metadata...
|
|
551
603
|
# {
|
|
@@ -570,47 +622,47 @@ def parse_acl(acl: Optional[Dict], bucket: str, aws_account_id: str) -> Optional
|
|
|
570
622
|
if acl is None:
|
|
571
623
|
return None
|
|
572
624
|
acl_list: List[Dict] = []
|
|
573
|
-
for grant in acl[
|
|
625
|
+
for grant in acl["Grants"]:
|
|
574
626
|
parsed_acl = None
|
|
575
|
-
if grant[
|
|
627
|
+
if grant["Grantee"]["Type"] == "CanonicalUser":
|
|
576
628
|
parsed_acl = {
|
|
577
629
|
"bucket": bucket,
|
|
578
|
-
"owner": acl[
|
|
579
|
-
"ownerid": acl[
|
|
580
|
-
"type": grant[
|
|
581
|
-
"displayname": grant[
|
|
582
|
-
"granteeid": grant[
|
|
630
|
+
"owner": acl["Owner"].get("DisplayName"),
|
|
631
|
+
"ownerid": acl["Owner"].get("ID"),
|
|
632
|
+
"type": grant["Grantee"]["Type"],
|
|
633
|
+
"displayname": grant["Grantee"].get("DisplayName"),
|
|
634
|
+
"granteeid": grant["Grantee"].get("ID"),
|
|
583
635
|
"uri": None,
|
|
584
|
-
"permission": grant.get(
|
|
636
|
+
"permission": grant.get("Permission"),
|
|
585
637
|
}
|
|
586
|
-
elif grant[
|
|
638
|
+
elif grant["Grantee"]["Type"] == "Group":
|
|
587
639
|
parsed_acl = {
|
|
588
640
|
"bucket": bucket,
|
|
589
|
-
"owner": acl[
|
|
590
|
-
"ownerid": acl[
|
|
591
|
-
"type": grant[
|
|
641
|
+
"owner": acl["Owner"].get("DisplayName"),
|
|
642
|
+
"ownerid": acl["Owner"].get("ID"),
|
|
643
|
+
"type": grant["Grantee"]["Type"],
|
|
592
644
|
"displayname": None,
|
|
593
645
|
"granteeid": None,
|
|
594
|
-
"uri": grant[
|
|
595
|
-
"permission": grant.get(
|
|
646
|
+
"uri": grant["Grantee"].get("URI"),
|
|
647
|
+
"permission": grant.get("Permission"),
|
|
596
648
|
}
|
|
597
649
|
else:
|
|
598
|
-
logger.warning("Unexpected grant type: %s", grant[
|
|
650
|
+
logger.warning("Unexpected grant type: %s", grant["Grantee"]["Type"])
|
|
599
651
|
continue
|
|
600
652
|
|
|
601
653
|
# TODO this can be replaced with a string join
|
|
602
654
|
id_data = "{}:{}:{}:{}:{}:{}:{}:{}".format(
|
|
603
655
|
aws_account_id,
|
|
604
|
-
parsed_acl[
|
|
605
|
-
parsed_acl[
|
|
606
|
-
parsed_acl[
|
|
607
|
-
parsed_acl[
|
|
608
|
-
parsed_acl[
|
|
609
|
-
parsed_acl[
|
|
610
|
-
parsed_acl[
|
|
656
|
+
parsed_acl["owner"],
|
|
657
|
+
parsed_acl["ownerid"],
|
|
658
|
+
parsed_acl["type"],
|
|
659
|
+
parsed_acl["displayname"],
|
|
660
|
+
parsed_acl["granteeid"],
|
|
661
|
+
parsed_acl["uri"],
|
|
662
|
+
parsed_acl["permission"],
|
|
611
663
|
)
|
|
612
664
|
|
|
613
|
-
parsed_acl[
|
|
665
|
+
parsed_acl["id"] = hashlib.sha256(id_data.encode("utf8")).hexdigest()
|
|
614
666
|
acl_list.append(parsed_acl)
|
|
615
667
|
|
|
616
668
|
return acl_list
|
|
@@ -618,7 +670,7 @@ def parse_acl(acl: Optional[Dict], bucket: str, aws_account_id: str) -> Optional
|
|
|
618
670
|
|
|
619
671
|
@timeit
|
|
620
672
|
def parse_encryption(bucket: str, encryption: Optional[Dict]) -> Optional[Dict]:
|
|
621
|
-
"""
|
|
673
|
+
"""Parses the S3 default encryption object and returns a dict of the relevant data"""
|
|
622
674
|
# Encryption object JSON looks like:
|
|
623
675
|
# {
|
|
624
676
|
# 'ServerSideEncryptionConfiguration': {
|
|
@@ -635,27 +687,29 @@ def parse_encryption(bucket: str, encryption: Optional[Dict]) -> Optional[Dict]:
|
|
|
635
687
|
# }
|
|
636
688
|
if encryption is None:
|
|
637
689
|
return None
|
|
638
|
-
_ssec = encryption.get(
|
|
690
|
+
_ssec = encryption.get("ServerSideEncryptionConfiguration", {})
|
|
639
691
|
# Rules is a list, but only one rule ever exists
|
|
640
692
|
try:
|
|
641
|
-
rule = _ssec.get(
|
|
693
|
+
rule = _ssec.get("Rules", []).pop()
|
|
642
694
|
except IndexError:
|
|
643
695
|
return None
|
|
644
|
-
algorithm = rule.get(
|
|
696
|
+
algorithm = rule.get("ApplyServerSideEncryptionByDefault", {}).get("SSEAlgorithm")
|
|
645
697
|
if not algorithm:
|
|
646
698
|
return None
|
|
647
699
|
return {
|
|
648
700
|
"bucket": bucket,
|
|
649
701
|
"default_encryption": True,
|
|
650
702
|
"encryption_algorithm": algorithm,
|
|
651
|
-
"encryption_key_id": rule.get("ApplyServerSideEncryptionByDefault", {}).get(
|
|
652
|
-
|
|
703
|
+
"encryption_key_id": rule.get("ApplyServerSideEncryptionByDefault", {}).get(
|
|
704
|
+
"KMSMasterKeyID",
|
|
705
|
+
),
|
|
706
|
+
"bucket_key_enabled": rule.get("BucketKeyEnabled"),
|
|
653
707
|
}
|
|
654
708
|
|
|
655
709
|
|
|
656
710
|
@timeit
|
|
657
711
|
def parse_versioning(bucket: str, versioning: Optional[Dict]) -> Optional[Dict]:
|
|
658
|
-
"""
|
|
712
|
+
"""Parses the S3 versioning object and returns a dict of the relevant data"""
|
|
659
713
|
# Versioning object JSON looks like:
|
|
660
714
|
# {
|
|
661
715
|
# 'Status': 'Enabled'|'Suspended',
|
|
@@ -671,8 +725,11 @@ def parse_versioning(bucket: str, versioning: Optional[Dict]) -> Optional[Dict]:
|
|
|
671
725
|
|
|
672
726
|
|
|
673
727
|
@timeit
|
|
674
|
-
def parse_public_access_block(
|
|
675
|
-
|
|
728
|
+
def parse_public_access_block(
|
|
729
|
+
bucket: str,
|
|
730
|
+
public_access_block: Optional[Dict],
|
|
731
|
+
) -> Optional[Dict]:
|
|
732
|
+
"""Parses the S3 public access block object and returns a dict of the relevant data"""
|
|
676
733
|
# Versioning object JSON looks like:
|
|
677
734
|
# {
|
|
678
735
|
# 'PublicAccessBlockConfiguration': {
|
|
@@ -695,7 +752,12 @@ def parse_public_access_block(bucket: str, public_access_block: Optional[Dict])
|
|
|
695
752
|
|
|
696
753
|
|
|
697
754
|
@timeit
|
|
698
|
-
def load_s3_buckets(
|
|
755
|
+
def load_s3_buckets(
|
|
756
|
+
neo4j_session: neo4j.Session,
|
|
757
|
+
data: Dict,
|
|
758
|
+
current_aws_account_id: str,
|
|
759
|
+
aws_update_tag: int,
|
|
760
|
+
) -> None:
|
|
699
761
|
ingest_bucket = """
|
|
700
762
|
MERGE (bucket:S3Bucket{id:$BucketName})
|
|
701
763
|
ON CREATE SET bucket.firstseen = timestamp(), bucket.creationdate = $CreationDate
|
|
@@ -726,19 +788,37 @@ def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict, current_aws_accoun
|
|
|
726
788
|
|
|
727
789
|
|
|
728
790
|
@timeit
|
|
729
|
-
def cleanup_s3_buckets(
|
|
730
|
-
|
|
791
|
+
def cleanup_s3_buckets(
|
|
792
|
+
neo4j_session: neo4j.Session,
|
|
793
|
+
common_job_parameters: Dict,
|
|
794
|
+
) -> None:
|
|
795
|
+
run_cleanup_job(
|
|
796
|
+
"aws_import_s3_buckets_cleanup.json",
|
|
797
|
+
neo4j_session,
|
|
798
|
+
common_job_parameters,
|
|
799
|
+
)
|
|
731
800
|
|
|
732
801
|
|
|
733
802
|
@timeit
|
|
734
|
-
def cleanup_s3_bucket_acl_and_policy(
|
|
735
|
-
|
|
803
|
+
def cleanup_s3_bucket_acl_and_policy(
|
|
804
|
+
neo4j_session: neo4j.Session,
|
|
805
|
+
common_job_parameters: Dict,
|
|
806
|
+
) -> None:
|
|
807
|
+
run_cleanup_job(
|
|
808
|
+
"aws_import_s3_acl_cleanup.json",
|
|
809
|
+
neo4j_session,
|
|
810
|
+
common_job_parameters,
|
|
811
|
+
)
|
|
736
812
|
|
|
737
813
|
|
|
738
814
|
@timeit
|
|
739
815
|
def sync(
|
|
740
|
-
neo4j_session: neo4j.Session,
|
|
741
|
-
|
|
816
|
+
neo4j_session: neo4j.Session,
|
|
817
|
+
boto3_session: boto3.session.Session,
|
|
818
|
+
regions: List[str],
|
|
819
|
+
current_aws_account_id: str,
|
|
820
|
+
update_tag: int,
|
|
821
|
+
common_job_parameters: Dict,
|
|
742
822
|
) -> None:
|
|
743
823
|
logger.info("Syncing S3 for account '%s'.", current_aws_account_id)
|
|
744
824
|
bucket_data = get_s3_bucket_list(boto3_session)
|
|
@@ -747,14 +827,19 @@ def sync(
|
|
|
747
827
|
cleanup_s3_buckets(neo4j_session, common_job_parameters)
|
|
748
828
|
|
|
749
829
|
acl_and_policy_data_iter = get_s3_bucket_details(boto3_session, bucket_data)
|
|
750
|
-
load_s3_details(
|
|
830
|
+
load_s3_details(
|
|
831
|
+
neo4j_session,
|
|
832
|
+
acl_and_policy_data_iter,
|
|
833
|
+
current_aws_account_id,
|
|
834
|
+
update_tag,
|
|
835
|
+
)
|
|
751
836
|
cleanup_s3_bucket_acl_and_policy(neo4j_session, common_job_parameters)
|
|
752
837
|
|
|
753
838
|
merge_module_sync_metadata(
|
|
754
839
|
neo4j_session,
|
|
755
|
-
group_type=
|
|
840
|
+
group_type="AWSAccount",
|
|
756
841
|
group_id=current_aws_account_id,
|
|
757
|
-
synced_type=
|
|
842
|
+
synced_type="S3Bucket",
|
|
758
843
|
update_tag=update_tag,
|
|
759
844
|
stat_handler=stat_handler,
|
|
760
845
|
)
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Dict
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
import boto3
|
|
6
|
+
import neo4j
|
|
7
|
+
from botocore.exceptions import ClientError
|
|
8
|
+
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.aws.s3.account_public_access_block import (
|
|
12
|
+
S3AccountPublicAccessBlockSchema,
|
|
13
|
+
)
|
|
14
|
+
from cartography.stats import get_stats_client
|
|
15
|
+
from cartography.util import aws_handle_regions
|
|
16
|
+
from cartography.util import merge_module_sync_metadata
|
|
17
|
+
from cartography.util import timeit
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
stat_handler = get_stats_client(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@timeit
|
|
24
|
+
@aws_handle_regions
|
|
25
|
+
def get_account_public_access_block(
|
|
26
|
+
boto3_session: boto3.session.Session, region: str, account_id: str
|
|
27
|
+
) -> List[Dict]:
|
|
28
|
+
"""
|
|
29
|
+
Get the S3 Account Public Access Block settings for a region.
|
|
30
|
+
Returns a list containing at most one item (the block configuration).
|
|
31
|
+
"""
|
|
32
|
+
client = boto3_session.client("s3control", region_name=region)
|
|
33
|
+
try:
|
|
34
|
+
# Use the provided account_id instead of retrieving it
|
|
35
|
+
response = client.get_public_access_block(AccountId=account_id)
|
|
36
|
+
return [response]
|
|
37
|
+
except ClientError as e:
|
|
38
|
+
if e.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration":
|
|
39
|
+
logger.warning(
|
|
40
|
+
f"No public access block configuration found for account {account_id} in region {region}"
|
|
41
|
+
)
|
|
42
|
+
return []
|
|
43
|
+
else:
|
|
44
|
+
raise
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def transform_account_public_access_block(
|
|
48
|
+
public_access_blocks: List[Dict],
|
|
49
|
+
region: str,
|
|
50
|
+
aws_account_id: str,
|
|
51
|
+
) -> List[Dict]:
|
|
52
|
+
"""
|
|
53
|
+
Transform S3 Account Public Access Block data for ingestion.
|
|
54
|
+
"""
|
|
55
|
+
transformed_data = []
|
|
56
|
+
|
|
57
|
+
for public_access_block in public_access_blocks:
|
|
58
|
+
pab = public_access_block.get("PublicAccessBlockConfiguration", {})
|
|
59
|
+
transformed = {
|
|
60
|
+
"id": f"{aws_account_id}:{region}",
|
|
61
|
+
"account_id": aws_account_id,
|
|
62
|
+
"region": region,
|
|
63
|
+
"block_public_acls": pab.get("BlockPublicAcls"),
|
|
64
|
+
"ignore_public_acls": pab.get("IgnorePublicAcls"),
|
|
65
|
+
"block_public_policy": pab.get("BlockPublicPolicy"),
|
|
66
|
+
"restrict_public_buckets": pab.get("RestrictPublicBuckets"),
|
|
67
|
+
}
|
|
68
|
+
transformed_data.append(transformed)
|
|
69
|
+
|
|
70
|
+
return transformed_data
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@timeit
|
|
74
|
+
def load_account_public_access_block(
|
|
75
|
+
neo4j_session: neo4j.Session,
|
|
76
|
+
data: List[Dict],
|
|
77
|
+
region: str,
|
|
78
|
+
aws_account_id: str,
|
|
79
|
+
update_tag: int,
|
|
80
|
+
) -> None:
|
|
81
|
+
"""
|
|
82
|
+
Load S3 Account Public Access Block information into the graph.
|
|
83
|
+
"""
|
|
84
|
+
logger.info(
|
|
85
|
+
f"Loading {len(data)} S3 Account Public Access Block configurations for region {region} into graph."
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
load(
|
|
89
|
+
neo4j_session,
|
|
90
|
+
S3AccountPublicAccessBlockSchema(),
|
|
91
|
+
data,
|
|
92
|
+
lastupdated=update_tag,
|
|
93
|
+
region=region,
|
|
94
|
+
AWS_ID=aws_account_id,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@timeit
|
|
99
|
+
def cleanup_account_public_access_block(
|
|
100
|
+
neo4j_session: neo4j.Session, common_job_parameters: Dict
|
|
101
|
+
) -> None:
|
|
102
|
+
"""
|
|
103
|
+
Run S3 Account Public Access Block cleanup job.
|
|
104
|
+
"""
|
|
105
|
+
logger.debug("Running S3 Account Public Access Block cleanup job.")
|
|
106
|
+
cleanup_job = GraphJob.from_node_schema(
|
|
107
|
+
S3AccountPublicAccessBlockSchema(), common_job_parameters
|
|
108
|
+
)
|
|
109
|
+
cleanup_job.run(neo4j_session)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@timeit
|
|
113
|
+
def sync(
|
|
114
|
+
neo4j_session: neo4j.Session,
|
|
115
|
+
boto3_session: boto3.session.Session,
|
|
116
|
+
regions: List[str],
|
|
117
|
+
current_aws_account_id: str,
|
|
118
|
+
update_tag: int,
|
|
119
|
+
common_job_parameters: Dict,
|
|
120
|
+
) -> None:
|
|
121
|
+
"""
|
|
122
|
+
Sync S3 Account Public Access Block settings for all regions.
|
|
123
|
+
"""
|
|
124
|
+
for region in regions:
|
|
125
|
+
logger.info(
|
|
126
|
+
f"Syncing S3 Account Public Access Block for {region} in account {current_aws_account_id}"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
public_access_blocks = get_account_public_access_block(
|
|
130
|
+
boto3_session, region, current_aws_account_id
|
|
131
|
+
)
|
|
132
|
+
transformed_data = transform_account_public_access_block(
|
|
133
|
+
public_access_blocks,
|
|
134
|
+
region,
|
|
135
|
+
current_aws_account_id,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Removed the unnecessary if check
|
|
139
|
+
load_account_public_access_block(
|
|
140
|
+
neo4j_session,
|
|
141
|
+
transformed_data,
|
|
142
|
+
region,
|
|
143
|
+
current_aws_account_id,
|
|
144
|
+
update_tag,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
cleanup_account_public_access_block(neo4j_session, common_job_parameters)
|
|
148
|
+
|
|
149
|
+
# Record that we've synced this module
|
|
150
|
+
merge_module_sync_metadata(
|
|
151
|
+
neo4j_session,
|
|
152
|
+
group_type="AWSAccount",
|
|
153
|
+
group_id=current_aws_account_id,
|
|
154
|
+
synced_type="S3AccountPublicAccessBlock",
|
|
155
|
+
update_tag=update_tag,
|
|
156
|
+
stat_handler=stat_handler,
|
|
157
|
+
)
|