dynamodictionary 1.0.3__tar.gz → 1.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ Metadata-Version: 2.4
2
+ Name: dynamodictionary
3
+ Version: 1.0.4
4
+ Summary: Dictionary like thing built on top of dynamodb
5
+ Home-page: https://github.com/jbylund/dynamodictionary
6
+ Author: Joe Bylund
7
+ Author-email: joseph.bylund@gmail.com
8
+ Requires-Dist: boto3
9
+ Provides-Extra: dev
10
+ Requires-Dist: black; extra == "dev"
11
+ Requires-Dist: isort; extra == "dev"
12
+ Requires-Dist: pre-commit; extra == "dev"
13
+ Requires-Dist: pylint; extra == "dev"
14
+ Requires-Dist: pytest; extra == "dev"
15
+ Dynamic: author
16
+ Dynamic: author-email
17
+ Dynamic: description
18
+ Dynamic: home-page
19
+ Dynamic: provides-extra
20
+ Dynamic: requires-dist
21
+ Dynamic: summary
22
+
23
+ Dictionary like thing built on top of dynamodb
@@ -0,0 +1,23 @@
1
+ Metadata-Version: 2.4
2
+ Name: dynamodictionary
3
+ Version: 1.0.4
4
+ Summary: Dictionary like thing built on top of dynamodb
5
+ Home-page: https://github.com/jbylund/dynamodictionary
6
+ Author: Joe Bylund
7
+ Author-email: joseph.bylund@gmail.com
8
+ Requires-Dist: boto3
9
+ Provides-Extra: dev
10
+ Requires-Dist: black; extra == "dev"
11
+ Requires-Dist: isort; extra == "dev"
12
+ Requires-Dist: pre-commit; extra == "dev"
13
+ Requires-Dist: pylint; extra == "dev"
14
+ Requires-Dist: pytest; extra == "dev"
15
+ Dynamic: author
16
+ Dynamic: author-email
17
+ Dynamic: description
18
+ Dynamic: home-page
19
+ Dynamic: provides-extra
20
+ Dynamic: requires-dist
21
+ Dynamic: summary
22
+
23
+ Dictionary like thing built on top of dynamodb
@@ -6,4 +6,8 @@ dynamodictionary.egg-info/dependency_links.txt
6
6
  dynamodictionary.egg-info/requires.txt
7
7
  dynamodictionary.egg-info/top_level.txt
8
8
  src/dynamodict/__init__.py
9
- src/dynamodict/dynamodict.py
9
+ src/dynamodict/dynamodict.py
10
+ src/dynamodict/serialization.py
11
+ src/dynamodict/utils.py
12
+ tests/test_dynamodictionary.py
13
+ tests/test_serialization.py
@@ -0,0 +1,8 @@
1
+ boto3
2
+
3
+ [dev]
4
+ black
5
+ isort
6
+ pre-commit
7
+ pylint
8
+ pytest
@@ -9,13 +9,21 @@ setuptools.setup(
9
9
  author="Joe Bylund",
10
10
  description=description,
11
11
  install_requires=[
12
- "cbor2", # TODO: do I want to just use pickle?
13
12
  "boto3",
14
13
  ],
14
+ extras_require={
15
+ "dev": [
16
+ "black",
17
+ "isort",
18
+ "pre-commit",
19
+ "pylint",
20
+ "pytest",
21
+ ],
22
+ },
15
23
  long_description=description,
16
24
  name="dynamodictionary",
17
25
  package_dir={"dynamodict": "src/dynamodict"},
18
26
  packages=["dynamodict"],
19
27
  url="https://github.com/jbylund/dynamodictionary",
20
- version="1.0.3",
28
+ version="1.0.4",
21
29
  )
@@ -0,0 +1,6 @@
1
+ from .dynamodict import DynamoDictionary, deserialize, serialize
2
+ from .utils import install_patches
3
+
4
+ __versioninfo__ = (1, 0, 4)
5
+ __version__ = ".".join(str(x) for x in __versioninfo__)
6
+ install_patches()
@@ -1,17 +1,21 @@
1
1
  #!/usr/bin/python
2
2
  """a class that acts a little like a dictionary that uses dynamo as a backing"""
3
+
3
4
  # std imports
4
5
  import base64
5
6
  import logging
7
+ import pickle
6
8
  import sys
7
9
  import time
10
+ from decimal import Decimal
8
11
 
9
12
  # site imports
10
13
  import boto3
11
14
  import botocore.exceptions
12
- import cbor2
13
15
 
14
16
  KEY_ATTR_NAME = "k"
17
+ KEY_DATA_TYPE = "B"
18
+ TTL_ATTR_NAME = "ttl"
15
19
  VAL_ATTR_NAME = "v"
16
20
 
17
21
  logging.basicConfig()
@@ -19,7 +23,7 @@ logger = logging.getLogger(__name__)
19
23
 
20
24
 
21
25
  def serialize(obj):
22
- return base64.b64encode(cbor2.dumps(obj))
26
+ return base64.b64encode(pickle.dumps(obj))
23
27
 
24
28
 
25
29
  def deserialize(obj):
@@ -27,7 +31,7 @@ def deserialize(obj):
27
31
  obj = obj.value
28
32
  except AttributeError:
29
33
  pass
30
- return cbor2.loads(base64.b64decode(obj))
34
+ return pickle.loads(base64.b64decode(obj))
31
35
 
32
36
 
33
37
  def is_permission_error(oops):
@@ -50,11 +54,12 @@ class DynamoDictionary(object):
50
54
  default_read_units = 25 // 2
51
55
  default_write_units = 25 // 2
52
56
 
53
- def __init__(self, table_name, read_units=None, write_units=None):
57
+ def __init__(self, table_name, read_units=None, write_units=None, ttl=None):
54
58
  self.table_name = table_name
55
- self.client = boto3.client("dynamodb", region_name="us-east-1")
56
59
  self.conn = boto3.resource("dynamodb", region_name="us-east-1")
60
+ self.client = self.conn.meta.client
57
61
  self.table = self.conn.Table(self.table_name)
62
+ self.ttl = ttl
58
63
  try:
59
64
  self.table.get_item(Key={KEY_ATTR_NAME: serialize("xxx")})
60
65
  except botocore.exceptions.ClientError as oops:
@@ -64,6 +69,20 @@ class DynamoDictionary(object):
64
69
  # check if it's a table does not exist error
65
70
  self.create_table(read_units=read_units, write_units=write_units)
66
71
  self.table.wait_until_exists()
72
+ try:
73
+ # ensure the table has ttl enabled and pointing to the right attribute
74
+ self.client.update_time_to_live(
75
+ TableName=self.table_name,
76
+ TimeToLiveSpecification={
77
+ "Enabled": True,
78
+ "AttributeName": TTL_ATTR_NAME,
79
+ },
80
+ )
81
+ except botocore.exceptions.ClientError as oops:
82
+ if "TimeToLive is already enabled" in str(oops):
83
+ pass
84
+ else:
85
+ raise
67
86
 
68
87
  def create_table(self, read_units=None, write_units=None):
69
88
  """create a table in case it doesn't exist"""
@@ -72,14 +91,14 @@ class DynamoDictionary(object):
72
91
  AttributeDefinitions=[
73
92
  {
74
93
  "AttributeName": KEY_ATTR_NAME,
75
- "AttributeType": "B",
76
- }
94
+ "AttributeType": KEY_DATA_TYPE,
95
+ },
77
96
  ],
78
97
  KeySchema=[
79
98
  {
80
99
  "AttributeName": KEY_ATTR_NAME,
81
100
  "KeyType": "HASH",
82
- }
101
+ },
83
102
  ],
84
103
  ProvisionedThroughput={
85
104
  "ReadCapacityUnits": read_units or self.default_read_units,
@@ -100,11 +119,16 @@ class DynamoDictionary(object):
100
119
 
101
120
  def __setitem__(self, key, value):
102
121
  """x.__setitem__(i, y) <==> x[i]=y"""
103
- skey = serialize(key)
104
- sval = serialize(value)
122
+ to_put = {
123
+ KEY_ATTR_NAME: serialize(key),
124
+ VAL_ATTR_NAME: serialize(value),
125
+ }
126
+ if self.ttl:
127
+ expiry = time.time() + self.ttl
128
+ to_put[TTL_ATTR_NAME] = Decimal(expiry)
105
129
  for i in range(5):
106
130
  try:
107
- self.table.put_item(Item={KEY_ATTR_NAME: skey, VAL_ATTR_NAME: sval})
131
+ self.table.put_item(Item=to_put)
108
132
  break
109
133
  except botocore.exceptions.ClientError as oops:
110
134
  if oops.response["Error"]["Code"] != "ProvisionedThroughputExceededException":
@@ -114,7 +138,7 @@ class DynamoDictionary(object):
114
138
  raise
115
139
  else:
116
140
  logger.info("Over rate limit for table %s, backing off!", self.table_name)
117
- time.sleep(0.1 * 2 ** i)
141
+ time.sleep(0.1 * 2**i)
118
142
  else:
119
143
  raise Exception("Tried 5 times could not write item!")
120
144
 
@@ -220,10 +244,16 @@ class DynamoDictionary(object):
220
244
  def multi_delete(self, keys):
221
245
  if not keys:
222
246
  return
247
+ for ikey in keys:
248
+ self.pop(ikey)
249
+ return
223
250
  # todo add retries and handle unprocessed items
224
- request_items = {self.table_name: [{"DeleteRequest": {"Key": {KEY_ATTR_NAME: {"B": serialize(key)}}}} for key in keys]}
251
+ table_operations = []
252
+ for ikey in keys:
253
+ item_delete_request = {"DeleteRequest": {"Key": {KEY_ATTR_NAME: {KEY_DATA_TYPE: serialize(ikey)}}}}
254
+ table_operations.append(item_delete_request)
225
255
  try:
226
- res = self.client.batch_write_item(RequestItems=request_items)
256
+ res = self.client.batch_write_item(RequestItems={self.table_name: table_operations})
227
257
  except botocore.exceptions.ClientError as oops:
228
258
  perm_error = is_permission_error(oops)
229
259
  if perm_error:
File without changes
@@ -0,0 +1,34 @@
1
+ import functools
2
+ import logging
3
+ import socket
4
+ import time
5
+
6
+ PATCHES_HOLDER = {}
7
+
8
+ logger = logging.getLogger(__name__)
9
+
10
+
11
+ def install_patches():
12
+ if PATCHES_HOLDER:
13
+ logger.info("Patches already installed...")
14
+ return
15
+ logger.info("Installing patches...")
16
+
17
+ PATCHES_HOLDER["getaddrinfo"] = real_getaddrinfo = socket.getaddrinfo
18
+
19
+ @functools.lru_cache(maxsize=2**12)
20
+ def cached_addrinfo(_timestamp_bust, *args, **kwargs):
21
+ return real_getaddrinfo(*args, **kwargs)
22
+
23
+ def getaddrinfo_with_caching(*args, **kwargs):
24
+ return cached_addrinfo(time.time() // (10 * 60), *args, **kwargs)
25
+
26
+ socket.getaddrinfo = getaddrinfo_with_caching
27
+
28
+
29
+ def remove_patches():
30
+ if not PATCHES_HOLDER:
31
+ logger.warning("Patches were not applied...")
32
+ return
33
+ logger.info("Removing patches...")
34
+ socket.getaddrinfo = PATCHES_HOLDER.pop("getaddrinfo")
@@ -0,0 +1,183 @@
1
+ import datetime
2
+ import os
3
+ import random
4
+ import sys
5
+ import unittest
6
+ from string import ascii_lowercase
7
+
8
+ import boto3
9
+
10
+ if True:
11
+ sys.path = ["./src"] + sys.path
12
+ import dynamodict
13
+
14
+ if "site-packages" in dynamodict.__file__:
15
+ raise AssertionError("import is wrong")
16
+
17
+
18
+ def randstr():
19
+ return "".join(random.choice(ascii_lowercase) for _ in range(20))
20
+
21
+
22
+ # CI sets a per-job prefix so leftover tables can be swept up even if a job is cancelled
23
+ TABLE_PREFIX = os.environ.get("TEST_TABLE_PREFIX", "test_")
24
+
25
+
26
+ def make_table_name():
27
+ return TABLE_PREFIX + randstr()
28
+
29
+
30
+ def delete_table_quietly(table_name):
31
+ """Delete a table, tolerating it being missing or still being created."""
32
+ client = boto3.client("dynamodb", region_name="us-east-1")
33
+ for _ in range(3):
34
+ try:
35
+ client.delete_table(TableName=table_name)
36
+ return
37
+ except client.exceptions.ResourceNotFoundException:
38
+ return
39
+ except client.exceptions.ResourceInUseException:
40
+ client.get_waiter("table_exists").wait(TableName=table_name)
41
+
42
+
43
+ class TestDynamodictionary(unittest.TestCase):
44
+ @classmethod
45
+ def setUpClass(cls):
46
+ cls.test_table_name = make_table_name()
47
+ # register cleanup first: it runs even if the constructor fails after creating the table
48
+ cls.addClassCleanup(delete_table_quietly, cls.test_table_name)
49
+ cls.mytable = dynamodict.DynamoDictionary(cls.test_table_name)
50
+
51
+ def setUp(self):
52
+ self.mytable.clear()
53
+
54
+ def tearDown(self):
55
+ self.mytable.clear()
56
+
57
+ def test_construction(self):
58
+ pass # was constructed in setUpClass
59
+
60
+ def test_coverage(self):
61
+ dynamodict_methods = [
62
+ x for x in dir(self.mytable) if hasattr(getattr(self.mytable, x), "__call__") and not x.startswith("_")
63
+ ]
64
+ test_methods = [x[5:] for x in dir(self) if x.startswith("test_")]
65
+ untested = set(dynamodict_methods) - set(test_methods)
66
+ if untested:
67
+ raise AssertionError("Following dynamodict methods are not tested: {}".format(", ".join(sorted(untested))))
68
+
69
+ def test_clear(self):
70
+ assert 0 == len(self.mytable)
71
+ for _ in range(100):
72
+ self.mytable[randstr()] = randstr()
73
+ assert 100 == len(self.mytable)
74
+ self.mytable.clear()
75
+ assert 0 == len(self.mytable)
76
+
77
+ def test_create_table(self):
78
+ pass
79
+
80
+ def test_drop_table(self):
81
+ pass
82
+
83
+ def test_get(self):
84
+ self.mytable["foo"] = "bar"
85
+ assert self.mytable.get("foo") == "bar"
86
+ self.mytable.clear()
87
+ assert self.mytable.get("foo") is None
88
+
89
+ def test_items(self):
90
+ keys = sorted(randstr() for _ in range(10))
91
+ for ikey in keys:
92
+ self.mytable[ikey] = ikey
93
+ assert sorted(self.mytable.items()) == sorted({ikey: ikey for ikey in keys}.items())
94
+
95
+ def test_iteritems(self):
96
+ keys = sorted(randstr() for _ in range(10))
97
+ for ikey in keys:
98
+ self.mytable[ikey] = "value_" + ikey
99
+ idx = 0
100
+ indices_seen = set()
101
+ for idx, obs in enumerate(self.mytable.iteritems()):
102
+ obskey, obsval = obs
103
+ obsidx = keys.index(obskey)
104
+ assert obskey == keys[obsidx]
105
+ assert obsval == "value_" + keys[obsidx]
106
+ indices_seen.add(obsidx)
107
+ assert 9 == idx
108
+ assert 10 == len(indices_seen)
109
+
110
+ def test_iterkeys(self):
111
+ keys = sorted(randstr() for _ in range(10))
112
+ for ikey in keys:
113
+ self.mytable[ikey] = 1
114
+ indexes = set()
115
+ for ikey in self.mytable.iterkeys():
116
+ indexes.add(keys.index(ikey))
117
+ assert len(indexes) == len(keys)
118
+
119
+ def test_itervalues(self):
120
+ unique_values = sorted(randstr() for _ in range(10))
121
+ for ival in unique_values:
122
+ self.mytable[randstr()] = ival
123
+ unseen_vals = set(unique_values)
124
+ for ival in self.mytable.values():
125
+ unseen_vals.remove(ival)
126
+ if unseen_vals:
127
+ raise AssertionError("There were values that were not seen")
128
+
129
+ def test_keys(self):
130
+ keys = sorted(randstr() for _ in range(10))
131
+ for ikey in keys:
132
+ self.mytable[ikey] = "v" + ikey
133
+ keys_back = self.mytable.keys()
134
+ assert isinstance(keys_back, set)
135
+ assert set(keys) == keys_back
136
+
137
+ def test_len(self):
138
+ assert 0 == len(self.mytable)
139
+ for _ in range(20):
140
+ self.mytable[randstr()] = randstr()
141
+ assert 20 == len(self.mytable)
142
+
143
+ def test_multi_delete(self):
144
+ keys_to_keep = sorted(randstr() for _ in range(10))
145
+ keys_to_drop = sorted(randstr() for _ in range(10))
146
+ for ikey in keys_to_keep + keys_to_drop:
147
+ self.mytable[ikey] = ikey
148
+ assert sorted(self.mytable) == sorted(keys_to_keep + keys_to_drop)
149
+ self.mytable.multi_delete(keys_to_drop)
150
+ assert sorted(self.mytable) == sorted(keys_to_keep)
151
+
152
+ def test_bool(self):
153
+ if self.mytable:
154
+ assert False
155
+ self.mytable["foo"] = "bar"
156
+ if not self.mytable:
157
+ raise False
158
+
159
+ def test_pop(self):
160
+ sentinel = object()
161
+ assert sentinel is self.mytable.pop(randstr(), sentinel)
162
+ with self.assertRaises(KeyError):
163
+ self.mytable.pop(randstr())
164
+ key = randstr()
165
+ self.mytable[key] = key
166
+ assert self.mytable.pop(key) == key
167
+ with self.assertRaises(KeyError):
168
+ self.mytable.pop(key)
169
+
170
+ def test_values(self):
171
+ """Values should give back the values as a list."""
172
+ keys = sorted(randstr() for _ in range(10))
173
+ for ikey in keys:
174
+ self.mytable["k" + ikey] = ikey
175
+ values = self.mytable.values()
176
+ assert isinstance(values, list)
177
+ assert sorted(values) == sorted(keys)
178
+
179
+ def test_ttl(self):
180
+ table_name = make_table_name()
181
+ self.addCleanup(delete_table_quietly, table_name)
182
+ table_with_ttl = dynamodict.DynamoDictionary(table_name, ttl=10)
183
+ table_with_ttl["foo"] = "bar"
@@ -0,0 +1,48 @@
1
+ import datetime
2
+ import sys
3
+ import unittest
4
+
5
+ if True:
6
+ sys.path = ["./src"] + sys.path
7
+ import dynamodict
8
+
9
+ if "site-packages" in dynamodict.__file__:
10
+ raise AssertionError("import is wrong")
11
+
12
+
13
+ class TestEncoder(unittest.TestCase):
14
+ def test_roundtrip_datetime(self):
15
+ test_obj = datetime.date.today()
16
+ rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
17
+ if rtd != test_obj:
18
+ raise AssertionError("xxx")
19
+
20
+ def test_roundtrip_dict(self):
21
+ test_obj = {(1, 2): 3, "foo": 1, ("foo", 1): "bar", "today": datetime.date.today()}
22
+ rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
23
+ if rtd != test_obj:
24
+ raise AssertionError("xxx")
25
+
26
+ def test_roundtrip_int(self):
27
+ test_obj = 1
28
+ rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
29
+ if rtd != test_obj:
30
+ raise AssertionError("xxx")
31
+
32
+ def test_roundtrip_list(self):
33
+ test_obj = ["foo", 1, None]
34
+ rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
35
+ if rtd != test_obj:
36
+ raise AssertionError("xxx")
37
+
38
+ def test_roundtrip_str(self):
39
+ test_obj = "foo"
40
+ rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
41
+ if rtd != test_obj:
42
+ raise AssertionError("xxx")
43
+
44
+ def test_roundtrip_tuple(self):
45
+ test_obj = (1, "foo", None)
46
+ rtd = dynamodict.deserialize(dynamodict.serialize(test_obj))
47
+ if rtd != test_obj:
48
+ raise AssertionError("xxx")
@@ -1,12 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: dynamodictionary
3
- Version: 1.0.3
4
- Summary: Dictionary like thing built on top of dynamodb
5
- Home-page: https://github.com/jbylund/dynamodictionary
6
- Author: Joe Bylund
7
- Author-email: joseph.bylund@gmail.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
-
11
- Dictionary like thing built on top of dynamodb
12
-
@@ -1,12 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: dynamodictionary
3
- Version: 1.0.3
4
- Summary: Dictionary like thing built on top of dynamodb
5
- Home-page: https://github.com/jbylund/dynamodictionary
6
- Author: Joe Bylund
7
- Author-email: joseph.bylund@gmail.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
-
11
- Dictionary like thing built on top of dynamodb
12
-
@@ -1,2 +0,0 @@
1
- cbor2
2
- boto3
@@ -1,4 +0,0 @@
1
- from .dynamodict import DynamoDictionary
2
-
3
- __versioninfo__ = (1, 0, 2)
4
- __version__ = ".".join(str(x) for x in __versioninfo__)