s3tk 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- s3tk/__init__.py +715 -0
- s3tk/checks.py +235 -0
- s3tk-0.5.0.dist-info/METADATA +682 -0
- s3tk-0.5.0.dist-info/RECORD +8 -0
- s3tk-0.5.0.dist-info/WHEEL +5 -0
- s3tk-0.5.0.dist-info/entry_points.txt +2 -0
- s3tk-0.5.0.dist-info/licenses/LICENSE.txt +22 -0
- s3tk-0.5.0.dist-info/top_level.txt +1 -0
s3tk/__init__.py
ADDED
|
@@ -0,0 +1,715 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import json
|
|
3
|
+
import fnmatch
|
|
4
|
+
from collections import Counter, OrderedDict
|
|
5
|
+
import warnings
|
|
6
|
+
import boto3
|
|
7
|
+
import botocore
|
|
8
|
+
import click
|
|
9
|
+
from joblib import Parallel, delayed
|
|
10
|
+
from .checks import AclCheck, PolicyCheck, PublicAccessCheck, LoggingCheck, VersioningCheck, EncryptionCheck, ObjectLoggingCheck
|
|
11
|
+
|
|
12
|
+
# fix for https://github.com/kennethreitz-archive/clint/issues/185
|
|
13
|
+
with warnings.catch_warnings():
|
|
14
|
+
warnings.simplefilter("ignore")
|
|
15
|
+
from clint.textui import colored, puts, indent
|
|
16
|
+
|
|
17
|
+
__version__ = '0.5.0'
|
|
18
|
+
|
|
19
|
+
canned_acls = [
|
|
20
|
+
{
|
|
21
|
+
'acl': 'private',
|
|
22
|
+
'grants': []
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
'acl': 'public-read',
|
|
26
|
+
'grants': [
|
|
27
|
+
{'Grantee': {'Type': 'Group', 'URI': 'http://acs.amazonaws.com/groups/global/AllUsers'}, 'Permission': 'READ'}
|
|
28
|
+
]
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
'acl': 'public-read-write',
|
|
32
|
+
'grants': [
|
|
33
|
+
{'Grantee': {'Type': 'Group', 'URI': 'http://acs.amazonaws.com/groups/global/AllUsers'}, 'Permission': 'READ'},
|
|
34
|
+
{'Grantee': {u'Type': 'Group', u'URI': 'http://acs.amazonaws.com/groups/global/AllUsers'}, 'Permission': 'WRITE'}
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
'acl': 'authenticated-read',
|
|
39
|
+
'grants': [
|
|
40
|
+
{'Grantee': {'Type': 'Group', 'URI': 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'}, 'Permission': 'READ'}
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
'acl': 'aws-exec-read',
|
|
45
|
+
'grants': [
|
|
46
|
+
{'Grantee': {'Type': 'CanonicalUser', 'DisplayName': 'za-team', 'ID': '6aa5a366c34c1cbe25dc49211496e913e0351eb0e8c37aa3477e40942ec6b97c'}, 'Permission': 'READ'}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
cached_s3 = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def s3():
|
|
55
|
+
# memoize
|
|
56
|
+
global cached_s3
|
|
57
|
+
if cached_s3 is None:
|
|
58
|
+
cached_s3 = boto3.resource('s3')
|
|
59
|
+
return cached_s3
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def notice(message):
|
|
63
|
+
puts(colored.yellow(message))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def abort(message):
|
|
67
|
+
puts(colored.red(message))
|
|
68
|
+
sys.exit(1)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def unicode_key(key):
|
|
72
|
+
return key
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def perform(check):
|
|
76
|
+
check.perform()
|
|
77
|
+
|
|
78
|
+
with indent(2):
|
|
79
|
+
if check.status == 'passed':
|
|
80
|
+
puts(colored.green('✔ ' + check.name + ' ' + unicode_key(check.pass_message)))
|
|
81
|
+
elif check.status == 'failed':
|
|
82
|
+
puts(colored.red('✘ ' + check.name + ' ' + check.fail_message))
|
|
83
|
+
else:
|
|
84
|
+
puts(colored.red('✘ ' + check.name + ' access denied'))
|
|
85
|
+
|
|
86
|
+
return check
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def fetch_buckets(buckets):
|
|
90
|
+
if buckets:
|
|
91
|
+
if any('*' in b for b in buckets):
|
|
92
|
+
return [b for b in s3().buckets.all() if any(fnmatch.fnmatch(b.name, bn) for bn in buckets)]
|
|
93
|
+
else:
|
|
94
|
+
return [s3().Bucket(bn) for bn in buckets]
|
|
95
|
+
else:
|
|
96
|
+
return s3().buckets.all()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def fix_check(klass, buckets, dry_run, fix_args={}):
|
|
100
|
+
for bucket in fetch_buckets(buckets):
|
|
101
|
+
check = klass(bucket)
|
|
102
|
+
check.perform()
|
|
103
|
+
|
|
104
|
+
if check.status == 'passed':
|
|
105
|
+
message = colored.green('already ' + check.pass_message)
|
|
106
|
+
elif check.status == 'denied':
|
|
107
|
+
message = colored.red('access denied')
|
|
108
|
+
else:
|
|
109
|
+
if dry_run:
|
|
110
|
+
message = colored.yellow('to be ' + check.pass_message)
|
|
111
|
+
else:
|
|
112
|
+
try:
|
|
113
|
+
check.fix(fix_args)
|
|
114
|
+
message = colored.blue('just ' + check.pass_message)
|
|
115
|
+
except botocore.exceptions.ClientError as e:
|
|
116
|
+
message = colored.red(str(e))
|
|
117
|
+
|
|
118
|
+
puts(bucket.name + ' ' + message)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def encrypt_object(bucket_name, key, dry_run, kms_key_id, customer_key):
|
|
122
|
+
obj = s3().Object(bucket_name, key)
|
|
123
|
+
str_key = unicode_key(key)
|
|
124
|
+
|
|
125
|
+
try:
|
|
126
|
+
if customer_key:
|
|
127
|
+
obj.load(SSECustomerAlgorithm='AES256', SSECustomerKey=customer_key)
|
|
128
|
+
|
|
129
|
+
encrypted = None
|
|
130
|
+
if customer_key:
|
|
131
|
+
encrypted = obj.sse_customer_algorithm is not None
|
|
132
|
+
elif kms_key_id:
|
|
133
|
+
encrypted = obj.server_side_encryption == 'aws:kms'
|
|
134
|
+
else:
|
|
135
|
+
encrypted = obj.server_side_encryption == 'AES256'
|
|
136
|
+
|
|
137
|
+
if encrypted:
|
|
138
|
+
puts(str_key + ' ' + colored.green('already encrypted'))
|
|
139
|
+
return 'already encrypted'
|
|
140
|
+
else:
|
|
141
|
+
if dry_run:
|
|
142
|
+
puts(str_key + ' ' + colored.yellow('to be encrypted'))
|
|
143
|
+
return 'to be encrypted'
|
|
144
|
+
else:
|
|
145
|
+
copy_source = {'Bucket': bucket_name, 'Key': obj.key}
|
|
146
|
+
|
|
147
|
+
# TODO support going from customer encryption to other forms
|
|
148
|
+
if kms_key_id:
|
|
149
|
+
obj.copy_from(
|
|
150
|
+
CopySource=copy_source,
|
|
151
|
+
ServerSideEncryption='aws:kms',
|
|
152
|
+
SSEKMSKeyId=kms_key_id
|
|
153
|
+
)
|
|
154
|
+
elif customer_key:
|
|
155
|
+
obj.copy_from(
|
|
156
|
+
CopySource=copy_source,
|
|
157
|
+
SSECustomerAlgorithm='AES256',
|
|
158
|
+
SSECustomerKey=customer_key
|
|
159
|
+
)
|
|
160
|
+
else:
|
|
161
|
+
obj.copy_from(
|
|
162
|
+
CopySource=copy_source,
|
|
163
|
+
ServerSideEncryption='AES256'
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
puts(str_key + ' ' + colored.blue('just encrypted'))
|
|
167
|
+
return 'just encrypted'
|
|
168
|
+
|
|
169
|
+
except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e:
|
|
170
|
+
puts(str_key + ' ' + colored.red(str(e)))
|
|
171
|
+
return 'error'
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def determine_mode(acl):
|
|
175
|
+
owner = acl.owner
|
|
176
|
+
grants = acl.grants
|
|
177
|
+
non_owner_grants = [grant for grant in grants if not (grant['Grantee'].get('ID') == owner['ID'] and grant['Permission'] == 'FULL_CONTROL')]
|
|
178
|
+
|
|
179
|
+
# TODO bucket-owner-read and bucket-owner-full-control
|
|
180
|
+
return next((ca['acl'] for ca in canned_acls if ca['grants'] == non_owner_grants), 'custom')
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def scan_object(bucket_name, key):
|
|
184
|
+
obj = s3().Object(bucket_name, key)
|
|
185
|
+
str_key = unicode_key(key)
|
|
186
|
+
|
|
187
|
+
try:
|
|
188
|
+
mode = determine_mode(obj.Acl())
|
|
189
|
+
|
|
190
|
+
if mode == 'private':
|
|
191
|
+
puts(str_key + ' ' + colored.green(mode))
|
|
192
|
+
else:
|
|
193
|
+
puts(str_key + ' ' + colored.yellow(mode))
|
|
194
|
+
|
|
195
|
+
return mode
|
|
196
|
+
except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e:
|
|
197
|
+
puts(str_key + ' ' + colored.red(str(e)))
|
|
198
|
+
return 'error'
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def reset_object(bucket_name, key, dry_run, acl):
|
|
202
|
+
obj = s3().Object(bucket_name, key)
|
|
203
|
+
str_key = unicode_key(key)
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
obj_acl = obj.Acl()
|
|
207
|
+
mode = determine_mode(obj_acl)
|
|
208
|
+
|
|
209
|
+
if mode == acl:
|
|
210
|
+
puts(str_key + ' ' + colored.green('ACL already ' + acl))
|
|
211
|
+
return 'ACL already ' + acl
|
|
212
|
+
elif dry_run:
|
|
213
|
+
puts(str_key + ' ' + colored.yellow('ACL to be updated to ' + acl))
|
|
214
|
+
return 'ACL to be updated to ' + acl
|
|
215
|
+
else:
|
|
216
|
+
obj_acl.put(ACL=acl)
|
|
217
|
+
puts(str_key + ' ' + colored.blue('ACL updated to ' + acl))
|
|
218
|
+
return 'ACL updated to ' + acl
|
|
219
|
+
|
|
220
|
+
except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e:
|
|
221
|
+
puts(str_key + ' ' + colored.red(str(e)))
|
|
222
|
+
return 'error'
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def delete_unencrypted_version(bucket_name, key, id, dry_run):
|
|
226
|
+
object_version = s3().ObjectVersion(bucket_name, key, id)
|
|
227
|
+
|
|
228
|
+
try:
|
|
229
|
+
obj = object_version.get()
|
|
230
|
+
if obj.get('ServerSideEncryption') or obj.get('SSECustomerAlgorithm'):
|
|
231
|
+
puts(key + ' ' + id + ' ' + colored.green('encrypted'))
|
|
232
|
+
return 'encrypted'
|
|
233
|
+
else:
|
|
234
|
+
if dry_run:
|
|
235
|
+
puts(key + ' ' + id + ' ' + colored.blue('to be deleted'))
|
|
236
|
+
return 'to be deleted'
|
|
237
|
+
else:
|
|
238
|
+
puts(key + ' ' + id + ' ' + colored.blue('deleted'))
|
|
239
|
+
object_version.delete()
|
|
240
|
+
return 'deleted'
|
|
241
|
+
except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e:
|
|
242
|
+
puts(key + ' ' + id + ' ' + colored.red(str(e)))
|
|
243
|
+
return 'error'
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def object_matches(key, only, _except):
|
|
247
|
+
match = True
|
|
248
|
+
|
|
249
|
+
if only:
|
|
250
|
+
match = fnmatch.fnmatch(key, only)
|
|
251
|
+
|
|
252
|
+
if _except and match:
|
|
253
|
+
match = not fnmatch.fnmatch(key, _except)
|
|
254
|
+
|
|
255
|
+
return match
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def parallelize(bucket, only, _except, fn, args=(), versions=False):
|
|
259
|
+
bucket = s3().Bucket(bucket)
|
|
260
|
+
|
|
261
|
+
# use prefix for performance
|
|
262
|
+
prefix = None
|
|
263
|
+
if only:
|
|
264
|
+
# get the first prefix before wildcard
|
|
265
|
+
prefix = '/'.join(only.split('*')[0].split('/')[:-1])
|
|
266
|
+
if prefix:
|
|
267
|
+
prefix = prefix + '/'
|
|
268
|
+
|
|
269
|
+
if versions:
|
|
270
|
+
object_versions = bucket.object_versions.filter(Prefix=prefix) if prefix else bucket.object_versions.all()
|
|
271
|
+
# delete markers have no size
|
|
272
|
+
return Parallel(n_jobs=24)(delayed(fn)(bucket.name, ov.object_key, ov.id, *args) for ov in object_versions if object_matches(ov.object_key, only, _except) and not ov.is_latest and ov.size is not None)
|
|
273
|
+
else:
|
|
274
|
+
objects = bucket.objects.filter(Prefix=prefix) if prefix else bucket.objects.all()
|
|
275
|
+
|
|
276
|
+
if only and '*' not in only:
|
|
277
|
+
objects = [s3().Object(bucket, only)]
|
|
278
|
+
|
|
279
|
+
return Parallel(n_jobs=24)(delayed(fn)(bucket.name, os.key, *args) for os in objects if object_matches(os.key, only, _except))
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def public_statement(bucket):
|
|
283
|
+
return OrderedDict([
|
|
284
|
+
('Sid', 'Public'),
|
|
285
|
+
('Effect', 'Allow'),
|
|
286
|
+
('Principal', '*'),
|
|
287
|
+
('Action', 's3:GetObject'),
|
|
288
|
+
('Resource', 'arn:aws:s3:::%s/*' % bucket.name)
|
|
289
|
+
])
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def no_object_acl_statement(bucket):
|
|
293
|
+
return OrderedDict([
|
|
294
|
+
('Sid', 'NoObjectAcl'),
|
|
295
|
+
('Effect', 'Deny'),
|
|
296
|
+
('Principal', '*'),
|
|
297
|
+
('Action', 's3:PutObjectAcl'),
|
|
298
|
+
('Resource', 'arn:aws:s3:::%s/*' % bucket.name)
|
|
299
|
+
])
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def public_uploads_statement(bucket):
|
|
303
|
+
return OrderedDict([
|
|
304
|
+
('Sid', 'PublicUploads'),
|
|
305
|
+
('Effect', 'Deny'),
|
|
306
|
+
('Principal', '*'),
|
|
307
|
+
('Action', ['s3:PutObject', 's3:PutObjectAcl']),
|
|
308
|
+
('Resource', 'arn:aws:s3:::%s/*' % bucket.name),
|
|
309
|
+
('Condition', {'StringNotEquals': {'s3:x-amz-acl': 'public-read'}})
|
|
310
|
+
])
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def no_uploads_statement(bucket):
|
|
314
|
+
return OrderedDict([
|
|
315
|
+
('Sid', 'NoUploads'),
|
|
316
|
+
('Effect', 'Deny'),
|
|
317
|
+
('Principal', '*'),
|
|
318
|
+
('Action', 's3:PutObject'),
|
|
319
|
+
('Resource', 'arn:aws:s3:::%s/*' % bucket.name)
|
|
320
|
+
])
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def encryption_statement(bucket):
|
|
324
|
+
return OrderedDict([
|
|
325
|
+
('Sid', 'Encryption'),
|
|
326
|
+
('Effect', 'Deny'),
|
|
327
|
+
('Principal', '*'),
|
|
328
|
+
('Action', 's3:PutObject'),
|
|
329
|
+
('Resource', 'arn:aws:s3:::%s/*' % bucket.name),
|
|
330
|
+
('Condition', {'StringNotEquals': {'s3:x-amz-server-side-encryption': 'AES256'}})
|
|
331
|
+
])
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def statement_matches(s1, s2):
|
|
335
|
+
s1 = dict(s1)
|
|
336
|
+
s2 = dict(s2)
|
|
337
|
+
s1.pop('Sid', None)
|
|
338
|
+
s2.pop('Sid', None)
|
|
339
|
+
return s1 == s2
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def fetch_policy(bucket):
|
|
343
|
+
policy = None
|
|
344
|
+
try:
|
|
345
|
+
policy = bucket.Policy().policy
|
|
346
|
+
except botocore.exceptions.ClientError as e:
|
|
347
|
+
if 'NoSuchBucket' not in str(e):
|
|
348
|
+
raise
|
|
349
|
+
|
|
350
|
+
if policy:
|
|
351
|
+
policy = json.loads(policy, object_pairs_hook=OrderedDict)
|
|
352
|
+
|
|
353
|
+
return policy
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def print_dns_bucket(name, buckets, found_buckets):
|
|
357
|
+
if name not in found_buckets:
|
|
358
|
+
puts(name)
|
|
359
|
+
with indent(2):
|
|
360
|
+
if name in buckets:
|
|
361
|
+
puts(colored.green('owned'))
|
|
362
|
+
else:
|
|
363
|
+
puts(colored.red('not owned'))
|
|
364
|
+
|
|
365
|
+
puts()
|
|
366
|
+
|
|
367
|
+
found_buckets.add(name)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def print_policy(policy):
|
|
371
|
+
with indent(2):
|
|
372
|
+
if any(policy['Statement']):
|
|
373
|
+
puts(colored.yellow(json.dumps(policy, indent=4)))
|
|
374
|
+
else:
|
|
375
|
+
puts(colored.yellow("None"))
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def summarize(values):
|
|
379
|
+
summary = Counter(values)
|
|
380
|
+
|
|
381
|
+
puts()
|
|
382
|
+
puts("Summary")
|
|
383
|
+
for k, v in summary.most_common():
|
|
384
|
+
puts(k + ': ' + str(v))
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def fetch_event_selectors():
|
|
388
|
+
# TODO get trails across all regions
|
|
389
|
+
# even regions without buckets may have multi-region trails
|
|
390
|
+
client = boto3.client('cloudtrail')
|
|
391
|
+
paginator = client.get_paginator('list_trails')
|
|
392
|
+
|
|
393
|
+
event_selectors = {}
|
|
394
|
+
for page in paginator.paginate():
|
|
395
|
+
for trail in page['Trails']:
|
|
396
|
+
name = trail['Name']
|
|
397
|
+
region_client = boto3.client('cloudtrail', region_name=trail['HomeRegion'])
|
|
398
|
+
response = region_client.get_event_selectors(TrailName=name)
|
|
399
|
+
for event_selector in response['EventSelectors']:
|
|
400
|
+
read_write_type = event_selector['ReadWriteType']
|
|
401
|
+
for data_resource in event_selector['DataResources']:
|
|
402
|
+
if data_resource['Type'] == 'AWS::S3::Object':
|
|
403
|
+
for value in data_resource['Values']:
|
|
404
|
+
if value == 'arn:aws:s3':
|
|
405
|
+
trail_response = region_client.get_trail(Name=name)['Trail']
|
|
406
|
+
if trail_response['IsMultiRegionTrail']:
|
|
407
|
+
bucket = ('global')
|
|
408
|
+
else:
|
|
409
|
+
bucket = ('region', trail['HomeRegion'])
|
|
410
|
+
path = ''
|
|
411
|
+
else:
|
|
412
|
+
parts = value.split("/", 2)
|
|
413
|
+
bucket = ('bucket', parts[0].replace('arn:aws:s3:::', ''))
|
|
414
|
+
path = parts[1]
|
|
415
|
+
if bucket not in event_selectors:
|
|
416
|
+
event_selectors[bucket] = []
|
|
417
|
+
event_selectors[bucket].append({'trail': name, 'path': path, 'read_write_type': read_write_type})
|
|
418
|
+
return event_selectors
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
@click.group()
|
|
422
|
+
@click.version_option(version=__version__)
|
|
423
|
+
def cli():
|
|
424
|
+
pass
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
@cli.command()
|
|
428
|
+
@click.argument('buckets', nargs=-1)
|
|
429
|
+
@click.option('--log-bucket', multiple=True, help='Check log bucket(s)')
|
|
430
|
+
@click.option('--log-prefix', help='Check log prefix')
|
|
431
|
+
@click.option('--skip-logging', is_flag=True, help='Skip logging check')
|
|
432
|
+
@click.option('--skip-versioning', is_flag=True, help='Skip versioning check')
|
|
433
|
+
@click.option('--skip-default-encryption', is_flag=True, help='Skip default encryption check')
|
|
434
|
+
@click.option('--default-encryption', is_flag=True) # no op, can't hide from help until click 7 released
|
|
435
|
+
@click.option('--object-level-logging', is_flag=True)
|
|
436
|
+
@click.option('--sns-topic', help='Send SNS notification for failures')
|
|
437
|
+
def scan(buckets, log_bucket=None, log_prefix=None, skip_logging=False, skip_versioning=False, skip_default_encryption=False, default_encryption=True, object_level_logging=False, sns_topic=None):
|
|
438
|
+
event_selectors = fetch_event_selectors() if object_level_logging else {}
|
|
439
|
+
|
|
440
|
+
checks = []
|
|
441
|
+
for bucket in fetch_buckets(buckets):
|
|
442
|
+
puts(bucket.name)
|
|
443
|
+
|
|
444
|
+
checks.append(perform(AclCheck(bucket)))
|
|
445
|
+
|
|
446
|
+
checks.append(perform(PolicyCheck(bucket)))
|
|
447
|
+
|
|
448
|
+
checks.append(perform(PublicAccessCheck(bucket)))
|
|
449
|
+
|
|
450
|
+
if not skip_logging:
|
|
451
|
+
checks.append(perform(LoggingCheck(bucket, log_bucket=log_bucket, log_prefix=log_prefix)))
|
|
452
|
+
|
|
453
|
+
if not skip_versioning:
|
|
454
|
+
checks.append(perform(VersioningCheck(bucket)))
|
|
455
|
+
|
|
456
|
+
if not skip_default_encryption:
|
|
457
|
+
checks.append(perform(EncryptionCheck(bucket)))
|
|
458
|
+
|
|
459
|
+
if object_level_logging:
|
|
460
|
+
checks.append(perform(ObjectLoggingCheck(bucket, event_selectors=event_selectors)))
|
|
461
|
+
|
|
462
|
+
puts()
|
|
463
|
+
|
|
464
|
+
failed_checks = [c for c in checks if c.status != 'passed']
|
|
465
|
+
if any(failed_checks):
|
|
466
|
+
if sns_topic:
|
|
467
|
+
topic = boto3.resource('sns').Topic(sns_topic)
|
|
468
|
+
message = ''
|
|
469
|
+
for check in failed_checks:
|
|
470
|
+
msg = check.fail_message if check.status == 'failed' else 'access denied'
|
|
471
|
+
message += check.bucket.name + ': ' + check.name + ' ' + msg + '\n'
|
|
472
|
+
topic.publish(Message=message, Subject='[s3tk] Scan Failures')
|
|
473
|
+
sys.exit(1)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
@cli.command(name='scan-dns')
|
|
477
|
+
def scan_dns():
|
|
478
|
+
buckets = set([b.name for b in s3().buckets.all()])
|
|
479
|
+
found_buckets = set()
|
|
480
|
+
|
|
481
|
+
client = boto3.client('route53')
|
|
482
|
+
paginator = client.get_paginator('list_hosted_zones')
|
|
483
|
+
|
|
484
|
+
for page in paginator.paginate():
|
|
485
|
+
for hosted_zone in page['HostedZones']:
|
|
486
|
+
paginator2 = client.get_paginator('list_resource_record_sets')
|
|
487
|
+
for page2 in paginator2.paginate(HostedZoneId=hosted_zone['Id']):
|
|
488
|
+
for resource_set in page2['ResourceRecordSets']:
|
|
489
|
+
if resource_set.get('AliasTarget'):
|
|
490
|
+
value = resource_set['AliasTarget']['DNSName']
|
|
491
|
+
if value.startswith('s3-website-') and value.endswith('.amazonaws.com.'):
|
|
492
|
+
print_dns_bucket(resource_set['Name'][:-1], buckets, found_buckets)
|
|
493
|
+
elif resource_set.get('ResourceRecords'):
|
|
494
|
+
for record in resource_set['ResourceRecords']:
|
|
495
|
+
value = record['Value']
|
|
496
|
+
if value.endswith('.s3.amazonaws.com'):
|
|
497
|
+
print_dns_bucket('.'.join(value.split('.')[:-3]), buckets, found_buckets)
|
|
498
|
+
if 's3-website-' in value and value.endswith('.amazonaws.com'):
|
|
499
|
+
print_dns_bucket(resource_set['Name'][:-1], buckets, found_buckets)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
@cli.command(name='block-public-access')
|
|
503
|
+
@click.argument('buckets', nargs=-1)
|
|
504
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
505
|
+
def block_public_access(buckets, dry_run=False):
|
|
506
|
+
if not buckets:
|
|
507
|
+
abort('Must specify at least one bucket or wildcard')
|
|
508
|
+
fix_check(PublicAccessCheck, buckets, dry_run)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
@cli.command(name='enable-logging')
|
|
512
|
+
@click.argument('buckets', nargs=-1)
|
|
513
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
514
|
+
@click.option('--log-bucket', required=True, help='Bucket to store logs')
|
|
515
|
+
@click.option('--log-prefix', help='Log prefix')
|
|
516
|
+
def enable_logging(buckets, log_bucket=None, log_prefix=None, dry_run=False):
|
|
517
|
+
fix_check(LoggingCheck, buckets, dry_run, {'log_bucket': log_bucket, 'log_prefix': log_prefix})
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
@cli.command(name='enable-versioning')
|
|
521
|
+
@click.argument('buckets', nargs=-1)
|
|
522
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
523
|
+
def enable_versioning(buckets, dry_run=False):
|
|
524
|
+
fix_check(VersioningCheck, buckets, dry_run)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
@cli.command(name='enable-default-encryption')
|
|
528
|
+
@click.argument('buckets', nargs=-1)
|
|
529
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
530
|
+
def enable_default_encryption(buckets, dry_run=False):
|
|
531
|
+
fix_check(EncryptionCheck, buckets, dry_run)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
@cli.command()
|
|
535
|
+
@click.argument('bucket')
|
|
536
|
+
@click.option('--only', help='Only certain objects')
|
|
537
|
+
@click.option('--except', '_except', help='Except certain objects')
|
|
538
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
539
|
+
@click.option('--kms-key-id', help='KMS key id')
|
|
540
|
+
@click.option('--customer-key', help='Customer key')
|
|
541
|
+
def encrypt(bucket, only=None, _except=None, dry_run=False, kms_key_id=None, customer_key=None):
|
|
542
|
+
summarize(parallelize(bucket, only, _except, encrypt_object, (dry_run, kms_key_id, customer_key,)))
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
@cli.command(name='scan-object-acl')
|
|
546
|
+
@click.argument('bucket')
|
|
547
|
+
@click.option('--only', help='Only certain objects')
|
|
548
|
+
@click.option('--except', '_except', help='Except certain objects')
|
|
549
|
+
def scan_object_acl(bucket, only=None, _except=None):
|
|
550
|
+
summarize(parallelize(bucket, only, _except, scan_object))
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
@cli.command(name='reset-object-acl')
|
|
554
|
+
@click.argument('bucket')
|
|
555
|
+
@click.option('--only', help='Only certain objects')
|
|
556
|
+
@click.option('--except', '_except', help='Except certain objects')
|
|
557
|
+
@click.option('--acl', default='private', help='ACL to use')
|
|
558
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
559
|
+
def reset_object_acl(bucket, only=None, _except=None, acl=None, dry_run=False):
|
|
560
|
+
summarize(parallelize(bucket, only, _except, reset_object, (dry_run, acl,)))
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
@cli.command(name='delete-unencrypted-versions')
|
|
564
|
+
@click.argument('bucket')
|
|
565
|
+
@click.option('--only', help='Only certain objects')
|
|
566
|
+
@click.option('--except', '_except', help='Except certain objects')
|
|
567
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
568
|
+
def delete_unencrypted_versions(bucket, only=None, _except=None, dry_run=False):
|
|
569
|
+
summarize(parallelize(bucket, only, _except, delete_unencrypted_version, (dry_run,), True))
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
@cli.command(name='list-policy')
|
|
573
|
+
@click.argument('buckets', nargs=-1)
|
|
574
|
+
@click.option('--named', is_flag=True, help='Print named statements')
|
|
575
|
+
def list_policy(buckets, named=False):
|
|
576
|
+
for bucket in fetch_buckets(buckets):
|
|
577
|
+
puts(bucket.name)
|
|
578
|
+
|
|
579
|
+
policy = fetch_policy(bucket)
|
|
580
|
+
|
|
581
|
+
with indent(2):
|
|
582
|
+
if policy is None:
|
|
583
|
+
puts(colored.yellow('None'))
|
|
584
|
+
else:
|
|
585
|
+
if named:
|
|
586
|
+
public = public_statement(bucket)
|
|
587
|
+
no_object_acl = no_object_acl_statement(bucket)
|
|
588
|
+
public_uploads = public_uploads_statement(bucket)
|
|
589
|
+
no_uploads = no_uploads_statement(bucket)
|
|
590
|
+
encryption = encryption_statement(bucket)
|
|
591
|
+
|
|
592
|
+
for statement in policy['Statement']:
|
|
593
|
+
if statement_matches(statement, public):
|
|
594
|
+
named_statement = 'Public'
|
|
595
|
+
elif statement_matches(statement, no_object_acl):
|
|
596
|
+
named_statement = 'No object ACL'
|
|
597
|
+
elif statement_matches(statement, public_uploads):
|
|
598
|
+
named_statement = 'Public uploads'
|
|
599
|
+
elif statement_matches(statement, no_uploads):
|
|
600
|
+
named_statement = 'No uploads'
|
|
601
|
+
elif statement_matches(statement, encryption):
|
|
602
|
+
named_statement = 'Encryption'
|
|
603
|
+
else:
|
|
604
|
+
named_statement = 'Custom'
|
|
605
|
+
|
|
606
|
+
puts(colored.yellow(named_statement))
|
|
607
|
+
|
|
608
|
+
else:
|
|
609
|
+
puts(colored.yellow(json.dumps(policy, indent=4)))
|
|
610
|
+
|
|
611
|
+
puts()
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
@cli.command(name='set-policy')
|
|
615
|
+
@click.argument('bucket')
|
|
616
|
+
@click.option('--public', is_flag=True, help='Make all objects public')
|
|
617
|
+
@click.option('--no-object-acl', is_flag=True, help='Prevent object ACL')
|
|
618
|
+
@click.option('--public-uploads', is_flag=True, help='Only public uploads')
|
|
619
|
+
@click.option('--no-uploads', is_flag=True, help='Prevent new uploads')
|
|
620
|
+
@click.option('--encryption', is_flag=True, help='Require encryption')
|
|
621
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
622
|
+
def set_policy(bucket, public=False, no_object_acl=False, public_uploads=False, no_uploads=False, encryption=False, dry_run=False):
|
|
623
|
+
bucket = s3().Bucket(bucket)
|
|
624
|
+
bucket_policy = bucket.Policy()
|
|
625
|
+
|
|
626
|
+
statements = []
|
|
627
|
+
|
|
628
|
+
if public:
|
|
629
|
+
statements.append(public_statement(bucket))
|
|
630
|
+
|
|
631
|
+
if no_object_acl:
|
|
632
|
+
statements.append(no_object_acl_statement(bucket))
|
|
633
|
+
|
|
634
|
+
if public_uploads:
|
|
635
|
+
statements.append(public_uploads_statement(bucket))
|
|
636
|
+
|
|
637
|
+
if no_uploads:
|
|
638
|
+
statements.append(no_uploads_statement(bucket))
|
|
639
|
+
|
|
640
|
+
if encryption:
|
|
641
|
+
statements.append(encryption_statement(bucket))
|
|
642
|
+
|
|
643
|
+
if any(statements):
|
|
644
|
+
puts('New policy')
|
|
645
|
+
policy = OrderedDict([
|
|
646
|
+
('Version', '2012-10-17'),
|
|
647
|
+
('Statement', statements)
|
|
648
|
+
])
|
|
649
|
+
print_policy(policy)
|
|
650
|
+
|
|
651
|
+
if not dry_run:
|
|
652
|
+
bucket_policy.put(Policy=json.dumps(policy))
|
|
653
|
+
else:
|
|
654
|
+
abort('No policies specified')
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
# experimental
|
|
658
|
+
@cli.command(name='update-policy')
|
|
659
|
+
@click.argument('bucket')
|
|
660
|
+
@click.option('--encryption/--no-encryption', default=None, help='Require encryption')
|
|
661
|
+
@click.option('--dry-run', is_flag=True, help='Dry run')
|
|
662
|
+
def update_policy(bucket, encryption=None, dry_run=False):
|
|
663
|
+
bucket = s3().Bucket(bucket)
|
|
664
|
+
|
|
665
|
+
policy = fetch_policy(bucket)
|
|
666
|
+
if not policy:
|
|
667
|
+
policy = OrderedDict([
|
|
668
|
+
('Version', '2012-10-17'),
|
|
669
|
+
('Statement', [])
|
|
670
|
+
])
|
|
671
|
+
|
|
672
|
+
es = encryption_statement(bucket)
|
|
673
|
+
es_index = next((i for i, s in enumerate(policy['Statement']) if statement_matches(s, es)), -1)
|
|
674
|
+
|
|
675
|
+
if es_index != -1:
|
|
676
|
+
if encryption:
|
|
677
|
+
puts("No encryption change")
|
|
678
|
+
print_policy(policy)
|
|
679
|
+
elif encryption is False:
|
|
680
|
+
puts("Removing encryption")
|
|
681
|
+
policy['Statement'].pop(es_index)
|
|
682
|
+
print_policy(policy)
|
|
683
|
+
|
|
684
|
+
if not dry_run:
|
|
685
|
+
if any(policy['Statement']):
|
|
686
|
+
bucket.Policy().put(Policy=json.dumps(policy))
|
|
687
|
+
else:
|
|
688
|
+
bucket.Policy().delete()
|
|
689
|
+
else:
|
|
690
|
+
if encryption:
|
|
691
|
+
puts("Adding encryption")
|
|
692
|
+
policy['Statement'].append(es)
|
|
693
|
+
print_policy(policy)
|
|
694
|
+
|
|
695
|
+
if not dry_run:
|
|
696
|
+
bucket.Policy().put(Policy=json.dumps(policy))
|
|
697
|
+
elif encryption is False:
|
|
698
|
+
puts(colored.yellow("No encryption change"))
|
|
699
|
+
print_policy(policy)
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
@cli.command(name='delete-policy')
|
|
703
|
+
@click.argument('bucket')
|
|
704
|
+
def delete_policy(bucket):
|
|
705
|
+
s3().Bucket(bucket).Policy().delete()
|
|
706
|
+
puts('Policy deleted')
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def main():
|
|
710
|
+
try:
|
|
711
|
+
cli()
|
|
712
|
+
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
|
|
713
|
+
abort(str(e))
|
|
714
|
+
except KeyboardInterrupt:
|
|
715
|
+
sys.exit(1)
|