dynamodictionary 1.0.1__tar.gz → 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dynamodictionary-1.0.4/PKG-INFO +23 -0
- {dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/README.md +17 -8
- dynamodictionary-1.0.4/dynamodictionary.egg-info/PKG-INFO +23 -0
- {dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/dynamodictionary.egg-info/SOURCES.txt +6 -1
- dynamodictionary-1.0.4/dynamodictionary.egg-info/requires.txt +8 -0
- dynamodictionary-1.0.4/setup.py +29 -0
- dynamodictionary-1.0.4/src/dynamodict/__init__.py +6 -0
- {dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/src/dynamodict/dynamodict.py +65 -53
- dynamodictionary-1.0.4/src/dynamodict/serialization.py +0 -0
- dynamodictionary-1.0.4/src/dynamodict/utils.py +34 -0
- dynamodictionary-1.0.4/tests/test_dynamodictionary.py +183 -0
- dynamodictionary-1.0.4/tests/test_serialization.py +48 -0
- dynamodictionary-1.0.1/PKG-INFO +0 -10
- dynamodictionary-1.0.1/dynamodictionary.egg-info/PKG-INFO +0 -10
- dynamodictionary-1.0.1/setup.py +0 -17
- dynamodictionary-1.0.1/src/dynamodict/__init__.py +0 -1
- {dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/dynamodictionary.egg-info/dependency_links.txt +0 -0
- {dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/dynamodictionary.egg-info/top_level.txt +0 -0
- {dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/setup.cfg +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dynamodictionary
|
|
3
|
+
Version: 1.0.4
|
|
4
|
+
Summary: Dictionary like thing built on top of dynamodb
|
|
5
|
+
Home-page: https://github.com/jbylund/dynamodictionary
|
|
6
|
+
Author: Joe Bylund
|
|
7
|
+
Author-email: joseph.bylund@gmail.com
|
|
8
|
+
Requires-Dist: boto3
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: black; extra == "dev"
|
|
11
|
+
Requires-Dist: isort; extra == "dev"
|
|
12
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
13
|
+
Requires-Dist: pylint; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest; extra == "dev"
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: author-email
|
|
17
|
+
Dynamic: description
|
|
18
|
+
Dynamic: home-page
|
|
19
|
+
Dynamic: provides-extra
|
|
20
|
+
Dynamic: requires-dist
|
|
21
|
+
Dynamic: summary
|
|
22
|
+
|
|
23
|
+
Dictionary like thing built on top of dynamodb
|
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
# dynamodictionary
|
|
2
|
+
|
|
3
|
+
## Installing
|
|
4
|
+
|
|
5
|
+
From github: `python -m pip install --upgrade git+https://git@github.com/jbylund/dynamodictionary.git`
|
|
6
|
+
|
|
7
|
+
From pypi: `python -m pip install --upgrade dynamodictionary`
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
## Example Use
|
|
2
11
|
|
|
3
12
|
This package provides a dictionary like interface for interacting with dynamodb tables, example:
|
|
4
13
|
|
|
@@ -9,26 +18,26 @@ In [2]: mytable = dynamodict.DynamoDictionary("footable")
|
|
|
9
18
|
|
|
10
19
|
In [3]: mytable['monty'] = 'python'
|
|
11
20
|
|
|
12
|
-
In [4]: print
|
|
21
|
+
In [4]: print(mytable['monty'])
|
|
13
22
|
python
|
|
14
23
|
|
|
15
24
|
In [5]: mytable['cheeses'] = ['applewood', 'brie', 'cheddar', 'duddleswell']
|
|
16
25
|
|
|
17
|
-
In [6]: print
|
|
26
|
+
In [6]: print(mytable['cheeses'])
|
|
18
27
|
['applewood', 'brie', 'cheddar', 'duddleswell']
|
|
19
28
|
|
|
20
29
|
In [7]: for key in footable:
|
|
21
|
-
...: print
|
|
30
|
+
...: print(key)
|
|
22
31
|
...:
|
|
23
32
|
cheeses
|
|
24
33
|
monty
|
|
25
34
|
|
|
26
|
-
In [8]: print
|
|
35
|
+
In [8]: print('cheeses' in footable)
|
|
27
36
|
True
|
|
28
37
|
|
|
29
|
-
In [9]: print
|
|
38
|
+
In [9]: print('watery tart' in footable)
|
|
30
39
|
False
|
|
31
40
|
|
|
32
|
-
In [10]: print
|
|
41
|
+
In [10]: print(len(footable))
|
|
33
42
|
2
|
|
34
|
-
```
|
|
43
|
+
```
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dynamodictionary
|
|
3
|
+
Version: 1.0.4
|
|
4
|
+
Summary: Dictionary like thing built on top of dynamodb
|
|
5
|
+
Home-page: https://github.com/jbylund/dynamodictionary
|
|
6
|
+
Author: Joe Bylund
|
|
7
|
+
Author-email: joseph.bylund@gmail.com
|
|
8
|
+
Requires-Dist: boto3
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: black; extra == "dev"
|
|
11
|
+
Requires-Dist: isort; extra == "dev"
|
|
12
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
13
|
+
Requires-Dist: pylint; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest; extra == "dev"
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: author-email
|
|
17
|
+
Dynamic: description
|
|
18
|
+
Dynamic: home-page
|
|
19
|
+
Dynamic: provides-extra
|
|
20
|
+
Dynamic: requires-dist
|
|
21
|
+
Dynamic: summary
|
|
22
|
+
|
|
23
|
+
Dictionary like thing built on top of dynamodb
|
|
@@ -3,6 +3,11 @@ setup.py
|
|
|
3
3
|
dynamodictionary.egg-info/PKG-INFO
|
|
4
4
|
dynamodictionary.egg-info/SOURCES.txt
|
|
5
5
|
dynamodictionary.egg-info/dependency_links.txt
|
|
6
|
+
dynamodictionary.egg-info/requires.txt
|
|
6
7
|
dynamodictionary.egg-info/top_level.txt
|
|
7
8
|
src/dynamodict/__init__.py
|
|
8
|
-
src/dynamodict/dynamodict.py
|
|
9
|
+
src/dynamodict/dynamodict.py
|
|
10
|
+
src/dynamodict/serialization.py
|
|
11
|
+
src/dynamodict/utils.py
|
|
12
|
+
tests/test_dynamodictionary.py
|
|
13
|
+
tests/test_serialization.py
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import setuptools
|
|
2
|
+
|
|
3
|
+
description = """
|
|
4
|
+
Dictionary like thing built on top of dynamodb
|
|
5
|
+
""".strip()
|
|
6
|
+
|
|
7
|
+
setuptools.setup(
|
|
8
|
+
author_email="joseph.bylund@gmail.com",
|
|
9
|
+
author="Joe Bylund",
|
|
10
|
+
description=description,
|
|
11
|
+
install_requires=[
|
|
12
|
+
"boto3",
|
|
13
|
+
],
|
|
14
|
+
extras_require={
|
|
15
|
+
"dev": [
|
|
16
|
+
"black",
|
|
17
|
+
"isort",
|
|
18
|
+
"pre-commit",
|
|
19
|
+
"pylint",
|
|
20
|
+
"pytest",
|
|
21
|
+
],
|
|
22
|
+
},
|
|
23
|
+
long_description=description,
|
|
24
|
+
name="dynamodictionary",
|
|
25
|
+
package_dir={"dynamodict": "src/dynamodict"},
|
|
26
|
+
packages=["dynamodict"],
|
|
27
|
+
url="https://github.com/jbylund/dynamodictionary",
|
|
28
|
+
version="1.0.4",
|
|
29
|
+
)
|
|
@@ -1,17 +1,21 @@
|
|
|
1
1
|
#!/usr/bin/python
|
|
2
2
|
"""a class that acts a little like a dictionary that uses dynamo as a backing"""
|
|
3
|
+
|
|
3
4
|
# std imports
|
|
4
5
|
import base64
|
|
5
6
|
import logging
|
|
7
|
+
import pickle
|
|
6
8
|
import sys
|
|
7
9
|
import time
|
|
10
|
+
from decimal import Decimal
|
|
8
11
|
|
|
9
12
|
# site imports
|
|
10
13
|
import boto3
|
|
11
14
|
import botocore.exceptions
|
|
12
|
-
import cbor2
|
|
13
15
|
|
|
14
16
|
KEY_ATTR_NAME = "k"
|
|
17
|
+
KEY_DATA_TYPE = "B"
|
|
18
|
+
TTL_ATTR_NAME = "ttl"
|
|
15
19
|
VAL_ATTR_NAME = "v"
|
|
16
20
|
|
|
17
21
|
logging.basicConfig()
|
|
@@ -19,7 +23,7 @@ logger = logging.getLogger(__name__)
|
|
|
19
23
|
|
|
20
24
|
|
|
21
25
|
def serialize(obj):
|
|
22
|
-
return base64.b64encode(
|
|
26
|
+
return base64.b64encode(pickle.dumps(obj))
|
|
23
27
|
|
|
24
28
|
|
|
25
29
|
def deserialize(obj):
|
|
@@ -27,14 +31,14 @@ def deserialize(obj):
|
|
|
27
31
|
obj = obj.value
|
|
28
32
|
except AttributeError:
|
|
29
33
|
pass
|
|
30
|
-
return
|
|
34
|
+
return pickle.loads(base64.b64decode(obj))
|
|
31
35
|
|
|
32
36
|
|
|
33
37
|
def is_permission_error(oops):
|
|
34
|
-
if getattr(oops,
|
|
38
|
+
if getattr(oops, "response", {}).get("Error", {}).get("Code") == "AccessDeniedException":
|
|
35
39
|
return {
|
|
36
|
-
|
|
37
|
-
|
|
40
|
+
"user": oops.response["Error"]["Message"].split()[1].partition("/")[-1],
|
|
41
|
+
"action": oops.operation_name,
|
|
38
42
|
}
|
|
39
43
|
return False
|
|
40
44
|
|
|
@@ -45,15 +49,17 @@ class PermissionError(Exception):
|
|
|
45
49
|
|
|
46
50
|
class DynamoDictionary(object):
|
|
47
51
|
"""a class that acts a little like a dictionary that uses dynamo as a backing"""
|
|
52
|
+
|
|
48
53
|
empty_sentinel = object()
|
|
49
54
|
default_read_units = 25 // 2
|
|
50
55
|
default_write_units = 25 // 2
|
|
51
56
|
|
|
52
|
-
def __init__(self, table_name, read_units=None, write_units=None):
|
|
57
|
+
def __init__(self, table_name, read_units=None, write_units=None, ttl=None):
|
|
53
58
|
self.table_name = table_name
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
59
|
+
self.conn = boto3.resource("dynamodb", region_name="us-east-1")
|
|
60
|
+
self.client = self.conn.meta.client
|
|
56
61
|
self.table = self.conn.Table(self.table_name)
|
|
62
|
+
self.ttl = ttl
|
|
57
63
|
try:
|
|
58
64
|
self.table.get_item(Key={KEY_ATTR_NAME: serialize("xxx")})
|
|
59
65
|
except botocore.exceptions.ClientError as oops:
|
|
@@ -63,6 +69,20 @@ class DynamoDictionary(object):
|
|
|
63
69
|
# check if it's a table does not exist error
|
|
64
70
|
self.create_table(read_units=read_units, write_units=write_units)
|
|
65
71
|
self.table.wait_until_exists()
|
|
72
|
+
try:
|
|
73
|
+
# ensure the table has ttl enabled and pointing to the right attribute
|
|
74
|
+
self.client.update_time_to_live(
|
|
75
|
+
TableName=self.table_name,
|
|
76
|
+
TimeToLiveSpecification={
|
|
77
|
+
"Enabled": True,
|
|
78
|
+
"AttributeName": TTL_ATTR_NAME,
|
|
79
|
+
},
|
|
80
|
+
)
|
|
81
|
+
except botocore.exceptions.ClientError as oops:
|
|
82
|
+
if "TimeToLive is already enabled" in str(oops):
|
|
83
|
+
pass
|
|
84
|
+
else:
|
|
85
|
+
raise
|
|
66
86
|
|
|
67
87
|
def create_table(self, read_units=None, write_units=None):
|
|
68
88
|
"""create a table in case it doesn't exist"""
|
|
@@ -70,20 +90,20 @@ class DynamoDictionary(object):
|
|
|
70
90
|
TableName=self.table_name,
|
|
71
91
|
AttributeDefinitions=[
|
|
72
92
|
{
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
}
|
|
93
|
+
"AttributeName": KEY_ATTR_NAME,
|
|
94
|
+
"AttributeType": KEY_DATA_TYPE,
|
|
95
|
+
},
|
|
76
96
|
],
|
|
77
97
|
KeySchema=[
|
|
78
98
|
{
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
99
|
+
"AttributeName": KEY_ATTR_NAME,
|
|
100
|
+
"KeyType": "HASH",
|
|
101
|
+
},
|
|
82
102
|
],
|
|
83
103
|
ProvisionedThroughput={
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
}
|
|
104
|
+
"ReadCapacityUnits": read_units or self.default_read_units,
|
|
105
|
+
"WriteCapacityUnits": write_units or self.default_write_units,
|
|
106
|
+
},
|
|
87
107
|
)
|
|
88
108
|
|
|
89
109
|
def drop_table(self):
|
|
@@ -93,20 +113,25 @@ class DynamoDictionary(object):
|
|
|
93
113
|
def __getitem__(self, key):
|
|
94
114
|
"""x.__getitem__(y) <==> x[y]"""
|
|
95
115
|
got = self.table.get_item(Key={KEY_ATTR_NAME: serialize(key)})
|
|
96
|
-
if
|
|
116
|
+
if "Item" not in got:
|
|
97
117
|
raise KeyError(key)
|
|
98
|
-
return deserialize(got[
|
|
118
|
+
return deserialize(got["Item"][VAL_ATTR_NAME])
|
|
99
119
|
|
|
100
120
|
def __setitem__(self, key, value):
|
|
101
121
|
"""x.__setitem__(i, y) <==> x[i]=y"""
|
|
102
|
-
|
|
103
|
-
|
|
122
|
+
to_put = {
|
|
123
|
+
KEY_ATTR_NAME: serialize(key),
|
|
124
|
+
VAL_ATTR_NAME: serialize(value),
|
|
125
|
+
}
|
|
126
|
+
if self.ttl:
|
|
127
|
+
expiry = time.time() + self.ttl
|
|
128
|
+
to_put[TTL_ATTR_NAME] = Decimal(expiry)
|
|
104
129
|
for i in range(5):
|
|
105
130
|
try:
|
|
106
|
-
self.table.put_item(Item=
|
|
131
|
+
self.table.put_item(Item=to_put)
|
|
107
132
|
break
|
|
108
133
|
except botocore.exceptions.ClientError as oops:
|
|
109
|
-
if oops.response[
|
|
134
|
+
if oops.response["Error"]["Code"] != "ProvisionedThroughputExceededException":
|
|
110
135
|
perm_error = is_permission_error(oops)
|
|
111
136
|
if perm_error:
|
|
112
137
|
raise PermissionError(perm_error)
|
|
@@ -127,23 +152,18 @@ class DynamoDictionary(object):
|
|
|
127
152
|
def pop(self, key, default=empty_sentinel):
|
|
128
153
|
"""D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\nIf key is not found, d is returned if given, otherwise KeyError is raised"""
|
|
129
154
|
try:
|
|
130
|
-
deleted = self.table.delete_item(
|
|
131
|
-
Key={KEY_ATTR_NAME: serialize(key)},
|
|
132
|
-
ReturnValues='ALL_OLD'
|
|
133
|
-
)
|
|
155
|
+
deleted = self.table.delete_item(Key={KEY_ATTR_NAME: serialize(key)}, ReturnValues="ALL_OLD")
|
|
134
156
|
except botocore.exceptions.ClientError as oops:
|
|
135
157
|
perm_error = is_permission_error(oops)
|
|
136
158
|
if perm_error:
|
|
137
159
|
raise PermissionError(perm_error)
|
|
138
160
|
raise
|
|
139
|
-
if
|
|
161
|
+
if "Attributes" not in deleted:
|
|
140
162
|
if default is not self.empty_sentinel:
|
|
141
163
|
return default
|
|
142
164
|
else:
|
|
143
165
|
raise KeyError(key)
|
|
144
|
-
return deserialize(
|
|
145
|
-
deleted['Attributes'][VAL_ATTR_NAME]
|
|
146
|
-
)
|
|
166
|
+
return deserialize(deleted["Attributes"][VAL_ATTR_NAME])
|
|
147
167
|
|
|
148
168
|
def iteritems(self):
|
|
149
169
|
"""D.iteritems() -> an iterator over the (key, value) items of D"""
|
|
@@ -151,20 +171,18 @@ class DynamoDictionary(object):
|
|
|
151
171
|
while True:
|
|
152
172
|
kwargs = {
|
|
153
173
|
"Limit": 1000,
|
|
154
|
-
"Select":
|
|
174
|
+
"Select": "ALL_ATTRIBUTES",
|
|
155
175
|
}
|
|
156
176
|
if start_key:
|
|
157
|
-
kwargs[
|
|
177
|
+
kwargs["ExclusiveStartKey"] = start_key
|
|
158
178
|
try:
|
|
159
|
-
response = self.table.scan(
|
|
160
|
-
**kwargs
|
|
161
|
-
)
|
|
179
|
+
response = self.table.scan(**kwargs)
|
|
162
180
|
except botocore.exceptions.ClientError as oops:
|
|
163
181
|
perm_error = is_permission_error(oops)
|
|
164
182
|
if perm_error:
|
|
165
183
|
raise PermissionError(perm_error)
|
|
166
184
|
raise oops
|
|
167
|
-
for item in response[
|
|
185
|
+
for item in response["Items"]:
|
|
168
186
|
yield deserialize(item[KEY_ATTR_NAME]), deserialize(item[VAL_ATTR_NAME])
|
|
169
187
|
start_key = response.get("LastEvaluatedKey")
|
|
170
188
|
if start_key is None:
|
|
@@ -176,7 +194,7 @@ class DynamoDictionary(object):
|
|
|
176
194
|
|
|
177
195
|
def keys(self):
|
|
178
196
|
"""D.keys() -> list of D's keys"""
|
|
179
|
-
return
|
|
197
|
+
return {key for key, _ in self.iteritems()}
|
|
180
198
|
|
|
181
199
|
def values(self):
|
|
182
200
|
"""D.values() -> list of D's values"""
|
|
@@ -226,22 +244,16 @@ class DynamoDictionary(object):
|
|
|
226
244
|
def multi_delete(self, keys):
|
|
227
245
|
if not keys:
|
|
228
246
|
return
|
|
247
|
+
for ikey in keys:
|
|
248
|
+
self.pop(ikey)
|
|
249
|
+
return
|
|
229
250
|
# todo add retries and handle unprocessed items
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
"Key": {
|
|
235
|
-
KEY_ATTR_NAME: {
|
|
236
|
-
"B": serialize(key)
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
} for key in keys
|
|
241
|
-
]
|
|
242
|
-
}
|
|
251
|
+
table_operations = []
|
|
252
|
+
for ikey in keys:
|
|
253
|
+
item_delete_request = {"DeleteRequest": {"Key": {KEY_ATTR_NAME: {KEY_DATA_TYPE: serialize(ikey)}}}}
|
|
254
|
+
table_operations.append(item_delete_request)
|
|
243
255
|
try:
|
|
244
|
-
res = self.client.batch_write_item(RequestItems=
|
|
256
|
+
res = self.client.batch_write_item(RequestItems={self.table_name: table_operations})
|
|
245
257
|
except botocore.exceptions.ClientError as oops:
|
|
246
258
|
perm_error = is_permission_error(oops)
|
|
247
259
|
if perm_error:
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import logging
|
|
3
|
+
import socket
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
PATCHES_HOLDER = {}
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def install_patches():
|
|
12
|
+
if PATCHES_HOLDER:
|
|
13
|
+
logger.info("Patches already installed...")
|
|
14
|
+
return
|
|
15
|
+
logger.info("Installing patches...")
|
|
16
|
+
|
|
17
|
+
PATCHES_HOLDER["getaddrinfo"] = real_getaddrinfo = socket.getaddrinfo
|
|
18
|
+
|
|
19
|
+
@functools.lru_cache(maxsize=2**12)
|
|
20
|
+
def cached_addrinfo(_timestamp_bust, *args, **kwargs):
|
|
21
|
+
return real_getaddrinfo(*args, **kwargs)
|
|
22
|
+
|
|
23
|
+
def getaddrinfo_with_caching(*args, **kwargs):
|
|
24
|
+
return cached_addrinfo(time.time() // (10 * 60), *args, **kwargs)
|
|
25
|
+
|
|
26
|
+
socket.getaddrinfo = getaddrinfo_with_caching
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def remove_patches():
|
|
30
|
+
if not PATCHES_HOLDER:
|
|
31
|
+
logger.warning("Patches were not applied...")
|
|
32
|
+
return
|
|
33
|
+
logger.info("Removing patches...")
|
|
34
|
+
socket.getaddrinfo = PATCHES_HOLDER.pop("getaddrinfo")
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import os
|
|
3
|
+
import random
|
|
4
|
+
import sys
|
|
5
|
+
import unittest
|
|
6
|
+
from string import ascii_lowercase
|
|
7
|
+
|
|
8
|
+
import boto3
|
|
9
|
+
|
|
10
|
+
if True:
|
|
11
|
+
sys.path = ["./src"] + sys.path
|
|
12
|
+
import dynamodict
|
|
13
|
+
|
|
14
|
+
if "site-packages" in dynamodict.__file__:
|
|
15
|
+
raise AssertionError("import is wrong")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def randstr():
|
|
19
|
+
return "".join(random.choice(ascii_lowercase) for _ in range(20))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# CI sets a per-job prefix so leftover tables can be swept up even if a job is cancelled
|
|
23
|
+
TABLE_PREFIX = os.environ.get("TEST_TABLE_PREFIX", "test_")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def make_table_name():
|
|
27
|
+
return TABLE_PREFIX + randstr()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def delete_table_quietly(table_name):
|
|
31
|
+
"""Delete a table, tolerating it being missing or still being created."""
|
|
32
|
+
client = boto3.client("dynamodb", region_name="us-east-1")
|
|
33
|
+
for _ in range(3):
|
|
34
|
+
try:
|
|
35
|
+
client.delete_table(TableName=table_name)
|
|
36
|
+
return
|
|
37
|
+
except client.exceptions.ResourceNotFoundException:
|
|
38
|
+
return
|
|
39
|
+
except client.exceptions.ResourceInUseException:
|
|
40
|
+
client.get_waiter("table_exists").wait(TableName=table_name)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TestDynamodictionary(unittest.TestCase):
|
|
44
|
+
@classmethod
|
|
45
|
+
def setUpClass(cls):
|
|
46
|
+
cls.test_table_name = make_table_name()
|
|
47
|
+
# register cleanup first: it runs even if the constructor fails after creating the table
|
|
48
|
+
cls.addClassCleanup(delete_table_quietly, cls.test_table_name)
|
|
49
|
+
cls.mytable = dynamodict.DynamoDictionary(cls.test_table_name)
|
|
50
|
+
|
|
51
|
+
def setUp(self):
|
|
52
|
+
self.mytable.clear()
|
|
53
|
+
|
|
54
|
+
def tearDown(self):
|
|
55
|
+
self.mytable.clear()
|
|
56
|
+
|
|
57
|
+
def test_construction(self):
|
|
58
|
+
pass # was constructed in setUpClass
|
|
59
|
+
|
|
60
|
+
def test_coverage(self):
|
|
61
|
+
dynamodict_methods = [
|
|
62
|
+
x for x in dir(self.mytable) if hasattr(getattr(self.mytable, x), "__call__") and not x.startswith("_")
|
|
63
|
+
]
|
|
64
|
+
test_methods = [x[5:] for x in dir(self) if x.startswith("test_")]
|
|
65
|
+
untested = set(dynamodict_methods) - set(test_methods)
|
|
66
|
+
if untested:
|
|
67
|
+
raise AssertionError("Following dynamodict methods are not tested: {}".format(", ".join(sorted(untested))))
|
|
68
|
+
|
|
69
|
+
def test_clear(self):
|
|
70
|
+
assert 0 == len(self.mytable)
|
|
71
|
+
for _ in range(100):
|
|
72
|
+
self.mytable[randstr()] = randstr()
|
|
73
|
+
assert 100 == len(self.mytable)
|
|
74
|
+
self.mytable.clear()
|
|
75
|
+
assert 0 == len(self.mytable)
|
|
76
|
+
|
|
77
|
+
def test_create_table(self):
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
def test_drop_table(self):
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
def test_get(self):
|
|
84
|
+
self.mytable["foo"] = "bar"
|
|
85
|
+
assert self.mytable.get("foo") == "bar"
|
|
86
|
+
self.mytable.clear()
|
|
87
|
+
assert self.mytable.get("foo") is None
|
|
88
|
+
|
|
89
|
+
def test_items(self):
|
|
90
|
+
keys = sorted(randstr() for _ in range(10))
|
|
91
|
+
for ikey in keys:
|
|
92
|
+
self.mytable[ikey] = ikey
|
|
93
|
+
assert sorted(self.mytable.items()) == sorted({ikey: ikey for ikey in keys}.items())
|
|
94
|
+
|
|
95
|
+
def test_iteritems(self):
|
|
96
|
+
keys = sorted(randstr() for _ in range(10))
|
|
97
|
+
for ikey in keys:
|
|
98
|
+
self.mytable[ikey] = "value_" + ikey
|
|
99
|
+
idx = 0
|
|
100
|
+
indices_seen = set()
|
|
101
|
+
for idx, obs in enumerate(self.mytable.iteritems()):
|
|
102
|
+
obskey, obsval = obs
|
|
103
|
+
obsidx = keys.index(obskey)
|
|
104
|
+
assert obskey == keys[obsidx]
|
|
105
|
+
assert obsval == "value_" + keys[obsidx]
|
|
106
|
+
indices_seen.add(obsidx)
|
|
107
|
+
assert 9 == idx
|
|
108
|
+
assert 10 == len(indices_seen)
|
|
109
|
+
|
|
110
|
+
def test_iterkeys(self):
|
|
111
|
+
keys = sorted(randstr() for _ in range(10))
|
|
112
|
+
for ikey in keys:
|
|
113
|
+
self.mytable[ikey] = 1
|
|
114
|
+
indexes = set()
|
|
115
|
+
for ikey in self.mytable.iterkeys():
|
|
116
|
+
indexes.add(keys.index(ikey))
|
|
117
|
+
assert len(indexes) == len(keys)
|
|
118
|
+
|
|
119
|
+
def test_itervalues(self):
|
|
120
|
+
unique_values = sorted(randstr() for _ in range(10))
|
|
121
|
+
for ival in unique_values:
|
|
122
|
+
self.mytable[randstr()] = ival
|
|
123
|
+
unseen_vals = set(unique_values)
|
|
124
|
+
for ival in self.mytable.values():
|
|
125
|
+
unseen_vals.remove(ival)
|
|
126
|
+
if unseen_vals:
|
|
127
|
+
raise AssertionError("There were values that were not seen")
|
|
128
|
+
|
|
129
|
+
def test_keys(self):
|
|
130
|
+
keys = sorted(randstr() for _ in range(10))
|
|
131
|
+
for ikey in keys:
|
|
132
|
+
self.mytable[ikey] = "v" + ikey
|
|
133
|
+
keys_back = self.mytable.keys()
|
|
134
|
+
assert isinstance(keys_back, set)
|
|
135
|
+
assert set(keys) == keys_back
|
|
136
|
+
|
|
137
|
+
def test_len(self):
|
|
138
|
+
assert 0 == len(self.mytable)
|
|
139
|
+
for _ in range(20):
|
|
140
|
+
self.mytable[randstr()] = randstr()
|
|
141
|
+
assert 20 == len(self.mytable)
|
|
142
|
+
|
|
143
|
+
def test_multi_delete(self):
|
|
144
|
+
keys_to_keep = sorted(randstr() for _ in range(10))
|
|
145
|
+
keys_to_drop = sorted(randstr() for _ in range(10))
|
|
146
|
+
for ikey in keys_to_keep + keys_to_drop:
|
|
147
|
+
self.mytable[ikey] = ikey
|
|
148
|
+
assert sorted(self.mytable) == sorted(keys_to_keep + keys_to_drop)
|
|
149
|
+
self.mytable.multi_delete(keys_to_drop)
|
|
150
|
+
assert sorted(self.mytable) == sorted(keys_to_keep)
|
|
151
|
+
|
|
152
|
+
def test_bool(self):
|
|
153
|
+
if self.mytable:
|
|
154
|
+
assert False
|
|
155
|
+
self.mytable["foo"] = "bar"
|
|
156
|
+
if not self.mytable:
|
|
157
|
+
raise False
|
|
158
|
+
|
|
159
|
+
def test_pop(self):
|
|
160
|
+
sentinel = object()
|
|
161
|
+
assert sentinel is self.mytable.pop(randstr(), sentinel)
|
|
162
|
+
with self.assertRaises(KeyError):
|
|
163
|
+
self.mytable.pop(randstr())
|
|
164
|
+
key = randstr()
|
|
165
|
+
self.mytable[key] = key
|
|
166
|
+
assert self.mytable.pop(key) == key
|
|
167
|
+
with self.assertRaises(KeyError):
|
|
168
|
+
self.mytable.pop(key)
|
|
169
|
+
|
|
170
|
+
def test_values(self):
|
|
171
|
+
"""Values should give back the values as a list."""
|
|
172
|
+
keys = sorted(randstr() for _ in range(10))
|
|
173
|
+
for ikey in keys:
|
|
174
|
+
self.mytable["k" + ikey] = ikey
|
|
175
|
+
values = self.mytable.values()
|
|
176
|
+
assert isinstance(values, list)
|
|
177
|
+
assert sorted(values) == sorted(keys)
|
|
178
|
+
|
|
179
|
+
def test_ttl(self):
|
|
180
|
+
table_name = make_table_name()
|
|
181
|
+
self.addCleanup(delete_table_quietly, table_name)
|
|
182
|
+
table_with_ttl = dynamodict.DynamoDictionary(table_name, ttl=10)
|
|
183
|
+
table_with_ttl["foo"] = "bar"
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import sys
|
|
3
|
+
import unittest
|
|
4
|
+
|
|
5
|
+
if True:
|
|
6
|
+
sys.path = ["./src"] + sys.path
|
|
7
|
+
import dynamodict
|
|
8
|
+
|
|
9
|
+
if "site-packages" in dynamodict.__file__:
|
|
10
|
+
raise AssertionError("import is wrong")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestEncoder(unittest.TestCase):
|
|
14
|
+
def test_roundtrip_datetime(self):
|
|
15
|
+
test_obj = datetime.date.today()
|
|
16
|
+
rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
|
|
17
|
+
if rtd != test_obj:
|
|
18
|
+
raise AssertionError("xxx")
|
|
19
|
+
|
|
20
|
+
def test_roundtrip_dict(self):
|
|
21
|
+
test_obj = {(1, 2): 3, "foo": 1, ("foo", 1): "bar", "today": datetime.date.today()}
|
|
22
|
+
rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
|
|
23
|
+
if rtd != test_obj:
|
|
24
|
+
raise AssertionError("xxx")
|
|
25
|
+
|
|
26
|
+
def test_roundtrip_int(self):
|
|
27
|
+
test_obj = 1
|
|
28
|
+
rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
|
|
29
|
+
if rtd != test_obj:
|
|
30
|
+
raise AssertionError("xxx")
|
|
31
|
+
|
|
32
|
+
def test_roundtrip_list(self):
|
|
33
|
+
test_obj = ["foo", 1, None]
|
|
34
|
+
rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
|
|
35
|
+
if rtd != test_obj:
|
|
36
|
+
raise AssertionError("xxx")
|
|
37
|
+
|
|
38
|
+
def test_roundtrip_str(self):
|
|
39
|
+
test_obj = "foo"
|
|
40
|
+
rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
|
|
41
|
+
if rtd != test_obj:
|
|
42
|
+
raise AssertionError("xxx")
|
|
43
|
+
|
|
44
|
+
def test_roundtrip_tuple(self):
|
|
45
|
+
test_obj = (1, "foo", None)
|
|
46
|
+
rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
|
|
47
|
+
if rtd != test_obj:
|
|
48
|
+
raise AssertionError("xxx")
|
dynamodictionary-1.0.1/PKG-INFO
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 1.0
|
|
2
|
-
Name: dynamodictionary
|
|
3
|
-
Version: 1.0.1
|
|
4
|
-
Summary: Dictionary like thing built on top of dynamodb
|
|
5
|
-
Home-page: https://github.com/jbylund/dynamodictionary
|
|
6
|
-
Author: Joe Bylund
|
|
7
|
-
Author-email: joseph.bylund@gmail.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Description: Dictionary like thing built on top of dynamodb
|
|
10
|
-
Platform: UNKNOWN
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 1.0
|
|
2
|
-
Name: dynamodictionary
|
|
3
|
-
Version: 1.0.1
|
|
4
|
-
Summary: Dictionary like thing built on top of dynamodb
|
|
5
|
-
Home-page: https://github.com/jbylund/dynamodictionary
|
|
6
|
-
Author: Joe Bylund
|
|
7
|
-
Author-email: joseph.bylund@gmail.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Description: Dictionary like thing built on top of dynamodb
|
|
10
|
-
Platform: UNKNOWN
|
dynamodictionary-1.0.1/setup.py
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import setuptools
|
|
2
|
-
|
|
3
|
-
description = """
|
|
4
|
-
Dictionary like thing built on top of dynamodb
|
|
5
|
-
""".strip()
|
|
6
|
-
|
|
7
|
-
setuptools.setup(
|
|
8
|
-
author_email='joseph.bylund@gmail.com',
|
|
9
|
-
author='Joe Bylund',
|
|
10
|
-
description=description,
|
|
11
|
-
long_description=description,
|
|
12
|
-
name='dynamodictionary',
|
|
13
|
-
package_dir={'dynamodict': 'src/dynamodict'},
|
|
14
|
-
packages=['dynamodict'],
|
|
15
|
-
url="https://github.com/jbylund/dynamodictionary",
|
|
16
|
-
version='1.0.1',
|
|
17
|
-
)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
from .dynamodict import DynamoDictionary
|
{dynamodictionary-1.0.1 → dynamodictionary-1.0.4}/dynamodictionary.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|