awsimple 3.10.0__py3-none-any.whl → 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of awsimple might be problematic. Click here for more details.
- awsimple/__init__.py +1 -0
- awsimple/__version__.py +1 -1
- awsimple/dynamodb.py +69 -58
- awsimple/exceptions.py +24 -0
- awsimple/pubsub.py +53 -17
- awsimple/s3.py +13 -12
- {awsimple-3.10.0.dist-info → awsimple-4.0.0.dist-info}/METADATA +1 -1
- awsimple-4.0.0.dist-info/RECORD +21 -0
- awsimple-3.10.0.dist-info/RECORD +0 -20
- {awsimple-3.10.0.dist-info → awsimple-4.0.0.dist-info}/WHEEL +0 -0
- {awsimple-3.10.0.dist-info → awsimple-4.0.0.dist-info}/licenses/LICENSE +0 -0
- {awsimple-3.10.0.dist-info → awsimple-4.0.0.dist-info}/licenses/LICENSE.txt +0 -0
- {awsimple-3.10.0.dist-info → awsimple-4.0.0.dist-info}/top_level.txt +0 -0
awsimple/__init__.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from .__version__ import __application_name__, __version__, __author__, __title__
|
|
2
2
|
from .mock import use_moto_mock_env_var, is_mock, use_localstack_env_var, is_using_localstack
|
|
3
|
+
from .exceptions import DynamoDBItemAlreadyExists, S3BucketAlreadyExistsNotOwnedByYou
|
|
3
4
|
from .aws import AWSAccess, AWSimpleException, boto_error_to_string
|
|
4
5
|
from .cache import get_disk_free, get_directory_size, lru_cache_write, CacheAccess, CACHE_DIR_ENV_VAR
|
|
5
6
|
from .dynamodb import DynamoDBAccess, dict_to_dynamodb, DBItemNotFound, DynamoDBTableNotFound, dynamodb_to_json, dynamodb_to_dict, QuerySelection, DictKey, convert_serializable_special_cases
|
awsimple/__version__.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
__application_name__ = "awsimple"
|
|
2
2
|
__title__ = __application_name__
|
|
3
3
|
__author__ = "abel"
|
|
4
|
-
__version__ = "
|
|
4
|
+
__version__ = "4.0.0"
|
|
5
5
|
__author_email__ = "j@abel.co"
|
|
6
6
|
__url__ = "https://github.com/jamesabel/awsimple"
|
|
7
7
|
__download_url__ = "https://github.com/jamesabel/awsimple"
|
awsimple/dynamodb.py
CHANGED
|
@@ -9,7 +9,7 @@ from collections import OrderedDict, defaultdict, namedtuple
|
|
|
9
9
|
import datetime
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
from os.path import getsize, getmtime
|
|
12
|
-
from typing import List, Union, Any, Type, Dict, Callable
|
|
12
|
+
from typing import List, Union, Any, Type, Dict, Callable
|
|
13
13
|
from pprint import pformat
|
|
14
14
|
from itertools import islice
|
|
15
15
|
import json
|
|
@@ -21,14 +21,13 @@ from collections.abc import Iterable
|
|
|
21
21
|
from logging import getLogger
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
import boto3
|
|
25
24
|
from botocore.exceptions import EndpointConnectionError, ClientError
|
|
26
25
|
from boto3.dynamodb.conditions import Key
|
|
27
26
|
from typeguard import typechecked
|
|
28
27
|
from dictim import dictim # type: ignore
|
|
29
|
-
from yasf import sf
|
|
30
28
|
|
|
31
|
-
from
|
|
29
|
+
from . import CacheAccess, __application_name__, AWSimpleException
|
|
30
|
+
from .exceptions import DynamoDBItemAlreadyExists
|
|
32
31
|
|
|
33
32
|
# don't require pillow, but convert images with it if it exists
|
|
34
33
|
pil_exists = False
|
|
@@ -47,6 +46,9 @@ handle_inexact_error = True
|
|
|
47
46
|
# for scan to dict
|
|
48
47
|
DictKey = namedtuple("DictKey", ["partition", "sort"]) # only for Primary Key with both partition and sort keys
|
|
49
48
|
|
|
49
|
+
metadata_table_name = f"__{__application_name__}_metadata__"
|
|
50
|
+
dynamodb_name = "dynamodb"
|
|
51
|
+
|
|
50
52
|
log = getLogger(__application_name__)
|
|
51
53
|
|
|
52
54
|
|
|
@@ -219,6 +221,10 @@ def _is_valid_db_pickled_file(file_path: Path, cache_life: Union[float, int, Non
|
|
|
219
221
|
|
|
220
222
|
|
|
221
223
|
class DynamoDBAccess(CacheAccess):
|
|
224
|
+
"""
|
|
225
|
+
AWS DynamoDB access
|
|
226
|
+
"""
|
|
227
|
+
|
|
222
228
|
@typechecked()
|
|
223
229
|
def __init__(self, table_name: Union[str, None] = None, **kwargs):
|
|
224
230
|
"""
|
|
@@ -232,8 +238,12 @@ class DynamoDBAccess(CacheAccess):
|
|
|
232
238
|
self.secondary_index_postfix = "-index"
|
|
233
239
|
|
|
234
240
|
self.table_name = table_name # can be None (the default) if we're only doing things that don't require a table name such as get_table_names()
|
|
235
|
-
|
|
236
|
-
|
|
241
|
+
|
|
242
|
+
# avoid recursion
|
|
243
|
+
if self.table_name is not None and self.table_name != metadata_table_name:
|
|
244
|
+
self.metadata_table = _DynamoDBMetadataTable(dynamodb_name, self.table_name, **kwargs)
|
|
245
|
+
else:
|
|
246
|
+
self.metadata_table = None
|
|
237
247
|
|
|
238
248
|
super().__init__(resource_name="dynamodb", **kwargs)
|
|
239
249
|
|
|
@@ -508,7 +518,8 @@ class DynamoDBAccess(CacheAccess):
|
|
|
508
518
|
created = True
|
|
509
519
|
except ClientError as e:
|
|
510
520
|
log.warning(e)
|
|
511
|
-
self.metadata_table
|
|
521
|
+
if self.metadata_table is not None:
|
|
522
|
+
self.metadata_table.update_table_mtime()
|
|
512
523
|
|
|
513
524
|
return created
|
|
514
525
|
|
|
@@ -729,9 +740,41 @@ class DynamoDBAccess(CacheAccess):
|
|
|
729
740
|
try:
|
|
730
741
|
table = self.resource.Table(self.table_name)
|
|
731
742
|
table.put_item(Item=item)
|
|
732
|
-
self.metadata_table
|
|
743
|
+
if self.metadata_table is not None:
|
|
744
|
+
self.metadata_table.update_table_mtime()
|
|
745
|
+
except self.client.exceptions.ResourceNotFoundException:
|
|
746
|
+
raise DynamoDBTableNotFound(str(self.table_name))
|
|
747
|
+
|
|
748
|
+
def put_item_if_not_exists(self, item: dict):
|
|
749
|
+
"""
|
|
750
|
+
Put (write) a DynamoDB table dict item if it doesn't already exist. If the item already exists, it is not modified and DynamoDBItemAlreadyExists is raised.
|
|
751
|
+
|
|
752
|
+
:param item: item
|
|
753
|
+
"""
|
|
754
|
+
assert self.resource is not None
|
|
755
|
+
primary_partition_key = self.get_primary_partition_key()
|
|
756
|
+
primary_sort_key = self.get_primary_sort_key()
|
|
757
|
+
try:
|
|
758
|
+
table = self.resource.Table(self.table_name)
|
|
759
|
+
if primary_sort_key is None:
|
|
760
|
+
table.put_item(Item=item, ConditionExpression="attribute_not_exists(#pk)", ExpressionAttributeNames={"#pk": primary_partition_key})
|
|
761
|
+
else:
|
|
762
|
+
table.put_item(
|
|
763
|
+
Item=item,
|
|
764
|
+
ConditionExpression="attribute_not_exists(#pk) AND attribute_not_exists(#sk)",
|
|
765
|
+
ExpressionAttributeNames={"#pk": primary_partition_key, "#sk": primary_sort_key},
|
|
766
|
+
)
|
|
767
|
+
if self.metadata_table is not None:
|
|
768
|
+
self.metadata_table.update_table_mtime()
|
|
769
|
+
already_exists = False
|
|
733
770
|
except self.client.exceptions.ResourceNotFoundException:
|
|
734
771
|
raise DynamoDBTableNotFound(str(self.table_name))
|
|
772
|
+
except self.client.exceptions.ConditionalCheckFailedException:
|
|
773
|
+
already_exists = True
|
|
774
|
+
if already_exists:
|
|
775
|
+
primary_partition_value = str(item.get(primary_partition_key, ""))
|
|
776
|
+
primary_sort_value = str(item.get(primary_sort_key, ""))
|
|
777
|
+
raise DynamoDBItemAlreadyExists(str(self.table_name), primary_partition_value, primary_sort_value)
|
|
735
778
|
|
|
736
779
|
# cant' do a @typechecked() since optional item requires a single type
|
|
737
780
|
def get_item(
|
|
@@ -886,6 +929,9 @@ class DynamoDBAccess(CacheAccess):
|
|
|
886
929
|
if len(filtered_row) > 0:
|
|
887
930
|
filtered_rows.append(filtered_row)
|
|
888
931
|
field_names = sorted(field_names_set)
|
|
932
|
+
primary_partition_key = self.get_primary_partition_key()
|
|
933
|
+
primary_sort_key = self.get_primary_sort_key()
|
|
934
|
+
filtered_rows.sort(key=lambda x: (x[primary_partition_key], x.get(primary_sort_key))) # sort by primary partition key, then sort key if it exists
|
|
889
935
|
|
|
890
936
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
891
937
|
with open(file_path, "w", newline="") as csvfile:
|
|
@@ -897,38 +943,21 @@ class DynamoDBAccess(CacheAccess):
|
|
|
897
943
|
log.info(f"wrote {len(filtered_rows)} rows to {file_path}")
|
|
898
944
|
|
|
899
945
|
|
|
900
|
-
|
|
901
|
-
metadata_table_name = f"__{__application_name__}_metadata__"
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
class _DynamoDBMetadataTable:
|
|
946
|
+
class _DynamoDBMetadataTable(DynamoDBAccess):
|
|
905
947
|
"""
|
|
906
948
|
Access metadata (e.g. most recent modification time) for DynamoDB tables.
|
|
907
949
|
"""
|
|
908
950
|
|
|
909
951
|
@typechecked()
|
|
910
|
-
def __init__(self,
|
|
911
|
-
self.
|
|
912
|
-
self.
|
|
952
|
+
def __init__(self, service: str, service_name: str, **kwargs):
|
|
953
|
+
self.service = service # DynamoDB, SNS, SQS, etc.
|
|
954
|
+
self.service_name = service_name # table name, queue name, topic name, etc.
|
|
955
|
+
self.primary_partition_key = "service" # e.g. "dynamodb", "sqs", "sns", etc.
|
|
913
956
|
self.primary_sort_key = "name" # e.g. table name
|
|
914
957
|
self.mtime_f_string = "mtime_f"
|
|
915
958
|
self.mtime_human_string = "mtime_human"
|
|
916
|
-
|
|
917
|
-
self.
|
|
918
|
-
self.resource = boto3.resource(self.service)
|
|
919
|
-
self.table = self.resource.Table(metadata_table_name)
|
|
920
|
-
|
|
921
|
-
def create_table(self):
|
|
922
|
-
key_schema = [{"AttributeName": self.primary_partition_key, "KeyType": "HASH"}, {"AttributeName": self.primary_sort_key, "KeyType": "RANGE"}] # Partition key # Sort key
|
|
923
|
-
|
|
924
|
-
attribute_definitions = [{"AttributeName": self.primary_partition_key, "AttributeType": "S"}, {"AttributeName": self.primary_sort_key, "AttributeType": "S"}]
|
|
925
|
-
|
|
926
|
-
self.table = self.resource.create_table(
|
|
927
|
-
TableName=metadata_table_name, KeySchema=key_schema, AttributeDefinitions=attribute_definitions, BillingMode="PAY_PER_REQUEST" # on-demand capacity mode
|
|
928
|
-
)
|
|
929
|
-
|
|
930
|
-
# Wait until the table exists
|
|
931
|
-
self.table.meta.client.get_waiter("table_exists").wait(TableName=metadata_table_name)
|
|
959
|
+
super().__init__(metadata_table_name, **kwargs)
|
|
960
|
+
self.create_table(partition_key=self.primary_partition_key, sort_key=self.primary_sort_key, partition_key_type=str, sort_key_type=str)
|
|
932
961
|
|
|
933
962
|
def update_table_mtime(self):
|
|
934
963
|
"""
|
|
@@ -938,27 +967,12 @@ class _DynamoDBMetadataTable:
|
|
|
938
967
|
item = dict_to_dynamodb(
|
|
939
968
|
{
|
|
940
969
|
self.primary_partition_key: self.service,
|
|
941
|
-
self.primary_sort_key: self.
|
|
970
|
+
self.primary_sort_key: self.service_name,
|
|
942
971
|
self.mtime_f_string: m_time,
|
|
943
972
|
self.mtime_human_string: datetime.datetime.fromtimestamp(m_time).astimezone().isoformat(),
|
|
944
973
|
}
|
|
945
974
|
)
|
|
946
|
-
|
|
947
|
-
retry_put = False
|
|
948
|
-
try:
|
|
949
|
-
self.table.put_item(Item=item)
|
|
950
|
-
except ClientError as e:
|
|
951
|
-
error_code = e.response["Error"]["Code"]
|
|
952
|
-
if error_code == "ValidationException":
|
|
953
|
-
self.client.delete_table(TableName=metadata_table_name)
|
|
954
|
-
self.table.meta.client.get_waiter("table_not_exists").wait(TableName=metadata_table_name)
|
|
955
|
-
if error_code == "ValidationException" or error_code == "ResourceNotFoundException":
|
|
956
|
-
self.create_table() # table doesn't exist, so create it
|
|
957
|
-
retry_put = True # and retry the put
|
|
958
|
-
else:
|
|
959
|
-
raise # some other exception, so re-raise it
|
|
960
|
-
if retry_put:
|
|
961
|
-
self.table.put_item(Item=item)
|
|
975
|
+
self.put_item(item=item)
|
|
962
976
|
|
|
963
977
|
@typechecked()
|
|
964
978
|
def get_table_mtime_f(self) -> Union[float | None]:
|
|
@@ -966,13 +980,10 @@ class _DynamoDBMetadataTable:
|
|
|
966
980
|
Get a table's mtime from the metadata table.
|
|
967
981
|
:return: table's mtime as a float or None if table hasn't been written to
|
|
968
982
|
"""
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
mtime_f = float(mtime_f_as_decimal) # boto3 uses Decimal for numbers, but we want a float
|
|
976
|
-
except DBItemNotFound:
|
|
977
|
-
pass
|
|
983
|
+
item = self.get_item(self.primary_partition_key, self.service, self.primary_sort_key, self.service_name)
|
|
984
|
+
if (mtime := item.get(self.mtime_f_string)) is None:
|
|
985
|
+
mtime_f = None
|
|
986
|
+
else:
|
|
987
|
+
mtime_f = float(mtime)
|
|
988
|
+
|
|
978
989
|
return mtime_f
|
awsimple/exceptions.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
class AWSimpleExceptionBase(Exception):
|
|
2
|
+
"""Base exception for AWSimple errors."""
|
|
3
|
+
|
|
4
|
+
pass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DynamoDBItemAlreadyExists(AWSimpleExceptionBase):
|
|
8
|
+
"""Raised when an item already exists in DynamoDB."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, table_name: str, primary_partition_value: str, primary_sort_value: str):
|
|
11
|
+
if len(primary_sort_value) < 1:
|
|
12
|
+
sort_message = ""
|
|
13
|
+
else:
|
|
14
|
+
sort_message = f" and {primary_sort_value=}"
|
|
15
|
+
message = f"Item with {primary_partition_value=}{sort_message} already exists in table {table_name}"
|
|
16
|
+
super().__init__(message)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class S3BucketAlreadyExistsNotOwnedByYou(AWSimpleExceptionBase):
|
|
20
|
+
"""Raised when an S3 bucket already exists and is not owned by you."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, bucket_name: str):
|
|
23
|
+
message = f'S3 Bucket "{bucket_name}" already exists and is not owned by you'
|
|
24
|
+
super().__init__(message)
|
awsimple/pubsub.py
CHANGED
|
@@ -3,7 +3,7 @@ pub/sub abstraction on top of AWS SNS and SQS using boto3.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import time
|
|
6
|
-
from typing import Any, Dict, List, Callable
|
|
6
|
+
from typing import Any, Dict, List, Callable, Union
|
|
7
7
|
from datetime import timedelta
|
|
8
8
|
from multiprocessing import Process, Queue, Event
|
|
9
9
|
from threading import Thread
|
|
@@ -24,9 +24,13 @@ log = Logger(__application_name__)
|
|
|
24
24
|
|
|
25
25
|
queue_timeout = timedelta(days=30).total_seconds()
|
|
26
26
|
|
|
27
|
+
sqs_name = "sqs"
|
|
28
|
+
|
|
27
29
|
|
|
28
30
|
@typechecked()
|
|
29
|
-
def remove_old_queues(
|
|
31
|
+
def remove_old_queues(
|
|
32
|
+
channel: str, profile_name: Union[str, None] = None, aws_access_key_id: Union[str, None] = None, aws_secret_access_key: Union[str, None] = None, region_name: Union[str, None] = None
|
|
33
|
+
) -> list[str]:
|
|
30
34
|
"""
|
|
31
35
|
Remove old SQS queues that have not been used recently.
|
|
32
36
|
"""
|
|
@@ -35,10 +39,12 @@ def remove_old_queues(channel: str) -> list[str]:
|
|
|
35
39
|
log.warning(f"blank channel ({channel=}) - not deleting any queues")
|
|
36
40
|
return removed
|
|
37
41
|
for sqs_queue_name in get_all_sqs_queues(channel):
|
|
38
|
-
sqs_metadata = _DynamoDBMetadataTable(
|
|
42
|
+
sqs_metadata = _DynamoDBMetadataTable(
|
|
43
|
+
sqs_name, sqs_queue_name, profile_name=profile_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name
|
|
44
|
+
)
|
|
39
45
|
mtime = sqs_metadata.get_table_mtime_f()
|
|
40
46
|
if mtime is not None and time.time() - mtime > queue_timeout:
|
|
41
|
-
sqs = SQSPollAccess(sqs_queue_name)
|
|
47
|
+
sqs = SQSPollAccess(sqs_queue_name, profile_name=profile_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name)
|
|
42
48
|
try:
|
|
43
49
|
sqs.delete_queue()
|
|
44
50
|
log.info(f'deleted "{sqs_queue_name}",{mtime=}')
|
|
@@ -49,20 +55,18 @@ def remove_old_queues(channel: str) -> list[str]:
|
|
|
49
55
|
|
|
50
56
|
|
|
51
57
|
@typechecked()
|
|
52
|
-
def _connect_sns_to_sqs(
|
|
58
|
+
def _connect_sns_to_sqs(sqs: SQSPollAccess, sns: SNSAccess) -> None:
|
|
53
59
|
"""
|
|
54
60
|
Connect an SQS queue to an SNS topic.
|
|
55
61
|
|
|
56
|
-
:param
|
|
57
|
-
:param
|
|
58
|
-
:param sqs: SQSPollAccess instance for the SQS queue.
|
|
62
|
+
:param sqs: SQS access object
|
|
63
|
+
:param sns: SNS access object
|
|
59
64
|
:return: None
|
|
60
65
|
"""
|
|
61
66
|
|
|
62
67
|
sqs_arn = sqs.get_arn()
|
|
63
68
|
|
|
64
69
|
# Find the topic by name
|
|
65
|
-
sns = SNSAccess(channel_name)
|
|
66
70
|
sns.create_topic()
|
|
67
71
|
topic_arn = sns.get_arn()
|
|
68
72
|
assert sns.resource is not None
|
|
@@ -71,7 +75,7 @@ def _connect_sns_to_sqs(channel_name: str, sqs_queue_name: str, sqs: SQSPollAcce
|
|
|
71
75
|
# Subscribe queue to topic
|
|
72
76
|
queue_arn = sqs.get_arn()
|
|
73
77
|
subscription = topic.subscribe(Protocol="sqs", Endpoint=queue_arn)
|
|
74
|
-
log.info(f"Subscribed {
|
|
78
|
+
log.info(f"Subscribed {sqs.queue_name} to topic {topic_arn}. Subscription ARN: {subscription.arn}")
|
|
75
79
|
|
|
76
80
|
# Update queue policy to allow SNS -> SQS
|
|
77
81
|
policy = {
|
|
@@ -90,7 +94,7 @@ def _connect_sns_to_sqs(channel_name: str, sqs_queue_name: str, sqs: SQSPollAcce
|
|
|
90
94
|
}
|
|
91
95
|
assert sqs.queue is not None
|
|
92
96
|
sqs.queue.set_attributes(Attributes={"Policy": json.dumps(policy)})
|
|
93
|
-
log.debug(f"Queue {
|
|
97
|
+
log.debug(f"Queue {sqs.queue_name} policy updated to allow topic {topic_arn}.")
|
|
94
98
|
|
|
95
99
|
|
|
96
100
|
class _SubscriptionThread(Thread):
|
|
@@ -118,7 +122,16 @@ class _SubscriptionThread(Thread):
|
|
|
118
122
|
class PubSub(Process):
|
|
119
123
|
|
|
120
124
|
@typechecked()
|
|
121
|
-
def __init__(
|
|
125
|
+
def __init__(
|
|
126
|
+
self,
|
|
127
|
+
channel: str,
|
|
128
|
+
node_name: str = get_node_name(),
|
|
129
|
+
sub_callback: Callable | None = None,
|
|
130
|
+
profile_name: Union[str, None] = None,
|
|
131
|
+
aws_access_key_id: Union[str, None] = None,
|
|
132
|
+
aws_secret_access_key: Union[str, None] = None,
|
|
133
|
+
region_name: Union[str, None] = None,
|
|
134
|
+
) -> None:
|
|
122
135
|
"""
|
|
123
136
|
Pub and Sub.
|
|
124
137
|
Create in a separate process to offload from main thread. Also facilitates use of moto mock in tests.
|
|
@@ -131,6 +144,11 @@ class PubSub(Process):
|
|
|
131
144
|
self.node_name = node_name # e.g., computer name
|
|
132
145
|
self.sub_callback = sub_callback
|
|
133
146
|
|
|
147
|
+
self.profile_name = profile_name
|
|
148
|
+
self.aws_access_key_id = aws_access_key_id
|
|
149
|
+
self.aws_secret_access_key = aws_secret_access_key
|
|
150
|
+
self.region_name = region_name
|
|
151
|
+
|
|
134
152
|
self._pub_queue = Queue() # type: Queue[Dict[str, Any]]
|
|
135
153
|
self._sub_queue = Queue() # type: Queue[str]
|
|
136
154
|
|
|
@@ -143,13 +161,31 @@ class PubSub(Process):
|
|
|
143
161
|
sqs_prefix = f"{self.channel}-"
|
|
144
162
|
sqs_queue_name = f"{sqs_prefix}{self.node_name}"
|
|
145
163
|
|
|
146
|
-
sns = SNSAccess(
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
164
|
+
sns = SNSAccess(
|
|
165
|
+
self.channel,
|
|
166
|
+
auto_create=True,
|
|
167
|
+
profile_name=self.profile_name,
|
|
168
|
+
aws_access_key_id=self.aws_access_key_id,
|
|
169
|
+
aws_secret_access_key=self.aws_secret_access_key,
|
|
170
|
+
region_name=self.region_name,
|
|
171
|
+
)
|
|
172
|
+
sqs_metadata = _DynamoDBMetadataTable(
|
|
173
|
+
sqs_name, sqs_queue_name, profile_name=self.profile_name, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, region_name=self.region_name
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
sqs = SQSPollAccess(
|
|
177
|
+
sqs_queue_name, profile_name=self.profile_name, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, region_name=self.region_name
|
|
178
|
+
)
|
|
150
179
|
if not sqs.exists():
|
|
151
180
|
sqs.create_queue()
|
|
152
|
-
|
|
181
|
+
sns = SNSAccess(
|
|
182
|
+
topic_name=self.channel,
|
|
183
|
+
profile_name=self.profile_name,
|
|
184
|
+
aws_access_key_id=self.aws_access_key_id,
|
|
185
|
+
aws_secret_access_key=self.aws_secret_access_key,
|
|
186
|
+
region_name=self.region_name,
|
|
187
|
+
)
|
|
188
|
+
_connect_sns_to_sqs(sqs, sns)
|
|
153
189
|
|
|
154
190
|
sqs_metadata.update_table_mtime() # update SQS use time (the existing infrastructure calls it a "table", but we're using it for the SQS queue)
|
|
155
191
|
remove_old_queues(sqs_prefix) # clean up old queues
|
awsimple/s3.py
CHANGED
|
@@ -23,7 +23,7 @@ from typeguard import typechecked
|
|
|
23
23
|
from hashy import get_string_sha512, get_file_sha512, get_bytes_sha512, get_dls_sha512
|
|
24
24
|
from yasf import sf
|
|
25
25
|
|
|
26
|
-
from awsimple import CacheAccess, __application_name__, lru_cache_write, AWSimpleException, convert_serializable_special_cases
|
|
26
|
+
from awsimple import CacheAccess, __application_name__, lru_cache_write, AWSimpleException, convert_serializable_special_cases, S3BucketAlreadyExistsNotOwnedByYou
|
|
27
27
|
|
|
28
28
|
# Use this project's name as a prefix to avoid string collisions. Use dashes instead of underscore since that's AWS's convention.
|
|
29
29
|
sha512_string = f"{__application_name__}-sha512"
|
|
@@ -518,17 +518,18 @@ class S3Access(CacheAccess):
|
|
|
518
518
|
else:
|
|
519
519
|
location = {"LocationConstraint": region}
|
|
520
520
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
521
|
+
try:
|
|
522
|
+
if self.public_readable:
|
|
523
|
+
self.client.create_bucket(Bucket=self.bucket_name, CreateBucketConfiguration=location, ACL="public-read")
|
|
524
|
+
else:
|
|
525
|
+
self.client.create_bucket(Bucket=self.bucket_name, CreateBucketConfiguration=location)
|
|
526
|
+
self.client.get_waiter("bucket_exists").wait(Bucket=self.bucket_name)
|
|
527
|
+
created = True
|
|
528
|
+
except self.client.exceptions.BucketAlreadyOwnedByYou:
|
|
529
|
+
created = False # already exists and owned by you
|
|
530
|
+
except self.client.exceptions.BucketAlreadyExists as e:
|
|
531
|
+
# bucket already exists and is owned by someone else
|
|
532
|
+
raise S3BucketAlreadyExistsNotOwnedByYou(str(self.bucket_name)) from e
|
|
532
533
|
return created
|
|
533
534
|
|
|
534
535
|
@typechecked()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
awsimple/__init__.py,sha256=RT9hATlcKnnvJYnSh7tCWEeeT2LyleL_vT1Zdz0nSkM,1051
|
|
2
|
+
awsimple/__version__.py,sha256=Bry6FGgVAiNxrCSmMuhVuRDUuAgjUcP10LSWqTEROD0,323
|
|
3
|
+
awsimple/aws.py,sha256=NbRu1v_J5j2-pcefNZ5Xggii3mM9nHpeHMz9L9K9r-U,7653
|
|
4
|
+
awsimple/cache.py,sha256=_LvPx76215t8KhAJOin6Pe2b4lWpB6kbKpdzgR4FeA4,7206
|
|
5
|
+
awsimple/dynamodb.py,sha256=7MNxAutOCMTS4JSX-DLOwzaImJ2TzIc7kfQzQPAy5y8,41193
|
|
6
|
+
awsimple/dynamodb_miv.py,sha256=4xPxQDYkIM-BVDGyAre6uqwJHsxguEbHbof8ztt-V7g,4645
|
|
7
|
+
awsimple/exceptions.py,sha256=Ew-S8YkHVWrZFI_Yik5n0cJ7Ss4Kig5JsEPQ-9z18SU,922
|
|
8
|
+
awsimple/logs.py,sha256=s9FhdDFWjfxGCVDx-FNTPgJ-YN1AOAgz4HNxTVfRARE,4108
|
|
9
|
+
awsimple/mock.py,sha256=eScbnxFF9xAosOAsL-NZgp_P-fezB6StQMkb85Y3TNo,574
|
|
10
|
+
awsimple/platform.py,sha256=TObvLIVgRGh-Mh4ZCxFfxAmrn8KNMHktr6XkDZX8JYE,376
|
|
11
|
+
awsimple/pubsub.py,sha256=AIWtY5PxTIwqol2l3FlSUPZaIQzkwywTiRboAUAVrgo,9403
|
|
12
|
+
awsimple/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
awsimple/s3.py,sha256=OhWF1uv4oLmBF5jAhKIi918iUQxx4CX8bTEvQ5wLYr8,24050
|
|
14
|
+
awsimple/sns.py,sha256=T_FyN8eSmBPo213HOfB3UBlMrvtBK768IaRo_ks-7do,3526
|
|
15
|
+
awsimple/sqs.py,sha256=9ZY7161CpmYpcxlCFIfW8bvMn9SGl4cgGR79I4MFLDk,17281
|
|
16
|
+
awsimple-4.0.0.dist-info/licenses/LICENSE,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
|
|
17
|
+
awsimple-4.0.0.dist-info/licenses/LICENSE.txt,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
|
|
18
|
+
awsimple-4.0.0.dist-info/METADATA,sha256=eVU_aKuAzloCJ7HMbidHYJSYrbfps9r3Oa5B60ESODI,6640
|
|
19
|
+
awsimple-4.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
20
|
+
awsimple-4.0.0.dist-info/top_level.txt,sha256=mwqCoH_8vAaK6iYioiRbasXmVCQM7AK3grNWOKp2VHE,9
|
|
21
|
+
awsimple-4.0.0.dist-info/RECORD,,
|
awsimple-3.10.0.dist-info/RECORD
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
awsimple/__init__.py,sha256=yo9COyuJZonzuW_5h2yvsQOtTqnwh8e_iMsMBqQxir0,964
|
|
2
|
-
awsimple/__version__.py,sha256=7f2QtdJhuSXcZ0FT7TWbKMnjn0kMNBF-iRvyVI8FxF8,324
|
|
3
|
-
awsimple/aws.py,sha256=NbRu1v_J5j2-pcefNZ5Xggii3mM9nHpeHMz9L9K9r-U,7653
|
|
4
|
-
awsimple/cache.py,sha256=_LvPx76215t8KhAJOin6Pe2b4lWpB6kbKpdzgR4FeA4,7206
|
|
5
|
-
awsimple/dynamodb.py,sha256=07Qz9j0GB5de5k6-OraDR3YCxTzIJa72jKGu5XWG9aY,40674
|
|
6
|
-
awsimple/dynamodb_miv.py,sha256=4xPxQDYkIM-BVDGyAre6uqwJHsxguEbHbof8ztt-V7g,4645
|
|
7
|
-
awsimple/logs.py,sha256=s9FhdDFWjfxGCVDx-FNTPgJ-YN1AOAgz4HNxTVfRARE,4108
|
|
8
|
-
awsimple/mock.py,sha256=eScbnxFF9xAosOAsL-NZgp_P-fezB6StQMkb85Y3TNo,574
|
|
9
|
-
awsimple/platform.py,sha256=TObvLIVgRGh-Mh4ZCxFfxAmrn8KNMHktr6XkDZX8JYE,376
|
|
10
|
-
awsimple/pubsub.py,sha256=HzwkjIwFicoV--30ctov8oVlKfXWe7eaLOF1dieV7qk,7692
|
|
11
|
-
awsimple/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
-
awsimple/s3.py,sha256=ydYLDj51b38lYI6LsJlc4i0PDj2vnWBuSkzdRKcWAUg,23858
|
|
13
|
-
awsimple/sns.py,sha256=T_FyN8eSmBPo213HOfB3UBlMrvtBK768IaRo_ks-7do,3526
|
|
14
|
-
awsimple/sqs.py,sha256=9ZY7161CpmYpcxlCFIfW8bvMn9SGl4cgGR79I4MFLDk,17281
|
|
15
|
-
awsimple-3.10.0.dist-info/licenses/LICENSE,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
|
|
16
|
-
awsimple-3.10.0.dist-info/licenses/LICENSE.txt,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
|
|
17
|
-
awsimple-3.10.0.dist-info/METADATA,sha256=u4tpQmqUL8lZXkDPRbKC83mgZ_Sk8fcK5h7EFOdMNxU,6641
|
|
18
|
-
awsimple-3.10.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
19
|
-
awsimple-3.10.0.dist-info/top_level.txt,sha256=mwqCoH_8vAaK6iYioiRbasXmVCQM7AK3grNWOKp2VHE,9
|
|
20
|
-
awsimple-3.10.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|