zlmdb 25.10.1__cp311-cp311-macosx_10_9_universal2.whl

zlmdb/tests/orm/_test_serialization.py

@@ -0,0 +1,144 @@
+###############################################################################
+#
+# The MIT License (MIT)
+#
+# Copyright (c) typedef int GmbH
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+###############################################################################
+
+import os
+import sys
+import timeit
+import uuid
+import platform
+import logging
+
+import humanize
+
+import txaio
+
+txaio.use_twisted()
+
+from zlmdb import _types  # noqa
+from _schema_fbs import User as UserFbs  # noqa
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+if sys.version_info >= (3, 6):
+    from _schema_py3 import User
+else:
+    from _schema_py2 import User
+
+_TEST = {"oid": 0, "uuid": None, "bytes": 0, "serializer": None}
+
+
+def _serializer_run_fbs():
+    serialize = _TEST["serializer"]._serialize_value
+    user = UserFbs.create_test_user()
+    data = serialize(user)
+    _TEST["bytes"] += len(data)
+
+
+def _serializer_run():
+    serialize = _TEST["serializer"]._serialize_value
+    user = User.create_test_user()
+    data = serialize(user)
+    _TEST["bytes"] += len(data)
+
+
+def _serialization_speed(serializer, testfun):
+    N = 10
+    M = 10000
+
+    samples = []
+
+    logging.info("running on:")
+    logging.info(sys.version)
+    logging.info(platform.uname())
+
+    _TEST["oid"] = 0
+    _TEST["uuid"] = uuid.uuid4()
+    _TEST["bytes"] = 0
+    _TEST["bytes"] = 0
+    _TEST["serializer"] = serializer
+
+    for i in range(N):
+        secs = timeit.timeit(testfun, number=M)
+        ops = round(float(M) / secs, 1)
+        samples.append(ops)
+        logging.info(
+            "{} objects/sec {}".format(ops, humanize.naturalsize(_TEST["bytes"]))
+        )
+
+    ops_max = max(samples)
+    bytes_per_obj = float(_TEST["bytes"]) / float(N * M)
+    logging.info(
+        "{} objects/sec max, {} bytes total, {} bytes/obj".format(
+            ops_max,
+            humanize.naturalsize(_TEST["bytes"]),
+            humanize.naturalsize(bytes_per_obj),
+        )
+    )
+
+    return ops_max, _TEST["bytes"]
+
+
+def test_json_serialization_speed():
+    ser = _types._JsonValuesMixin(marshal=User.marshal, unmarshal=User.parse)
+    ops_max, total = _serialization_speed(ser, _serializer_run)
+    # cpy36: 19564.6 objects/sec max, 135456153 bytes total
+    assert ops_max > 1000
+    assert total > 1000000
+
+
+def test_cbor_serialization_speed():
+    ser = _types._CborValuesMixin(marshal=User.marshal, unmarshal=User.parse)
+    ops_max, total = _serialization_speed(ser, _serializer_run)
+    # cpy36: 7787.4 objects/sec max, 97815364 bytes total
+    assert ops_max > 1000
+    assert total > 1000000
+
+
+def test_pickle_serialization_speed():
+    ser = _types._PickleValuesMixin()
+    ops_max, total = _serialization_speed(ser, _serializer_run)
+    # cpy36: 33586.0 objects/sec max, 137738869 bytes total
+    assert ops_max > 1000
+    assert total > 1000000
+
+
+def test_flatbuffer_serialization_speed():
+    ser = _types._FlatBuffersValuesMixin(build=UserFbs.build, cast=UserFbs.cast)
+    ops_max, total = _serialization_speed(ser, _serializer_run_fbs)
+    assert ops_max > 1000
+    assert total > 1000000
+
+
+if __name__ == "__main__":
+    from typing import List
+
+    sers: List[object] = []
+    sers.append(_types._JsonValuesMixin(marshal=User.marshal, unmarshal=User.parse))
+    sers.append(_types._CborValuesMixin(marshal=User.marshal, unmarshal=User.parse))
+    sers.append(_types._PickleValuesMixin())
+    sers.append(_types._FlatBuffersValuesMixin(build=UserFbs.build, cast=UserFbs.cast))
+    for ser in sers:
+        logging.info(_serialization_speed(ser, _serializer_run))

zlmdb/tests/orm/test_basic.py

@@ -0,0 +1,217 @@
+###############################################################################
+#
+# The MIT License (MIT)
+#
+# Copyright (c) typedef int GmbH
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+###############################################################################
+
+import sys
+import os
+import pytest
+import logging
+
+import txaio
+
+txaio.use_twisted()
+
+import zlmdb  # noqa
+
+try:
+    from tempfile import TemporaryDirectory
+except ImportError:
+    from backports.tempfile import TemporaryDirectory  # type:ignore
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+if sys.version_info >= (3, 6):
+    from _schema_py3 import User, Schema2
+else:
+    from _schema_py2 import User, Schema2
+
+
+@pytest.fixture(scope="module")
+def testset1():
+    users = []
+    for i in range(1000):
+        user = User.create_test_user(i)
+        users.append(user)
+    return users
+
+
+def test_open1():
+    with TemporaryDirectory() as dbpath:
+        with zlmdb.Database(dbpath) as db:
+            assert db.maxsize == 10485760
+            assert db.is_sync
+            assert db.is_open
+            assert not db.is_readonly
+            assert not db.is_writemap
+
+
+def test_open2():
+    with TemporaryDirectory() as dbpath:
+        with zlmdb.Database(dbpath) as db:
+            assert db.is_open
+        with zlmdb.Database(dbpath) as db:
+            assert db.is_open
+
+
+def test_open3():
+    with TemporaryDirectory() as dbpath:
+        db = zlmdb.Database(dbpath)
+        assert db.is_open
+        try:
+            zlmdb.Database(dbpath)
+            assert False, (
+                "opening same dbpath twice in same process did not throw an exception"
+            )
+        except RuntimeError as e:
+            # RuntimeError: tried to open same dbpath "/tmp/tmpwc1dw5c8" twice within same process
+            assert "twice within same process" in str(e), (
+                "exception did not contain text we excepted: {}".format(e)
+            )
+        except Exception as e:
+            assert False, "unexpected exception {} raised".format(e)
+
+
+def test_open4():
+    with TemporaryDirectory() as dbpath:
+        db1 = zlmdb.Database.open(dbpath)
+        assert db1.is_open
+
+        db2 = zlmdb.Database.open(dbpath)
+        assert db2.is_open
+
+        assert db1 == db2
+
+
+def test_transaction():
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                logging.info("transaction open {}".format(txn.id()))
+            logging.info("transaction committed")
+        logging.info("database closed")
+
+
+def test_save_load():
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema2()
+
+        user = User.create_test_user()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                schema.users[txn, user.oid] = user
+                logging.info("user saved")
+
+                _user = schema.users[txn, user.oid]
+                assert _user
+                assert user == _user
+                logging.info("user loaded")
+
+            logging.info("transaction committed")
+
+        logging.info("database closed")
+
+
+def test_save_load_many_1(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema2()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.oid] = user
+
+                cnt = schema.users.count(txn)
+                logging.info("user saved: {}".format(cnt))
+                assert cnt == len(testset1)
+
+            with db.begin() as txn:
+                cnt = schema.users.count(txn)
+                assert cnt == len(testset1)
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                cnt = schema.users.count(txn)
+                assert cnt == len(testset1)
+
+
+def test_save_load_many_2(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema2()
+
+        oids = []
+
+        with zlmdb.Database(dbpath) as db:
+            # write records in a 1st transaction
+            with db.begin(write=True) as txn:
+                c = 0
+                for user in testset1:
+                    schema.users[txn, user.oid] = user
+                    oids.append(user.oid)
+                    c += 1
+                assert c == len(testset1)
+                logging.info("[1] successfully stored {} records".format(c))
+
+                # in the same transaction, read back records
+                c = 0
+                for oid in oids:
+                    user = schema.users[txn, oid]
+                    if user:
+                        c += 1
+                assert c == len(testset1)
+                logging.info("[1] successfully loaded {} records".format(c))
+
+            # in a new transaction, read back records
+            c = 0
+            with db.begin() as txn:
+                for oid in oids:
+                    user = schema.users[txn, oid]
+                    if user:
+                        c += 1
+            assert c == len(testset1)
+            logging.info("[2] successfully loaded {} records".format(c))
+
+        # in a new database environment (and hence new transaction), read back records
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                count = schema.users.count(txn)
+                assert count == len(testset1)
+                logging.info("total records: {}".format(count))
+
+                c = 0
+                for oid in oids:
+                    user = schema.users[txn, oid]
+                    if user:
+                        c += 1
+                assert c == len(testset1)
+                logging.info("[3] successfully loaded {} records".format(c))

zlmdb/tests/orm/test_etcd.py

@@ -0,0 +1,275 @@
+###############################################################################
+#
+# The MIT License (MIT)
+#
+# Copyright (c) typedef int GmbH
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+###############################################################################
+
+import os
+import sys
+import pytest
+import logging
+
+import txaio
+
+txaio.use_twisted()
+
+import zlmdb  # noqa
+
+try:
+    from tempfile import TemporaryDirectory
+except ImportError:
+    from backports.tempfile import TemporaryDirectory  # type:ignore
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+if sys.version_info >= (3, 6):
+    from _schema_py3 import User, Schema1, Schema3, Schema4
+else:
+    from _schema_py2 import User, Schema1, Schema3, Schema4
+
+
+@pytest.fixture(scope="module")
+def testset1():
+    users = []
+    for i in range(1000):
+        user = User.create_test_user(i)
+        users.append(user)
+    return users
+
+
+def test_truncate_table():
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema1()
+
+        stats = zlmdb.TransactionStats()
+        tabs = [
+            schema.tab_oid_json,
+            schema.tab_str_json,
+            schema.tab_uuid_json,
+            schema.tab_oid_cbor,
+            schema.tab_str_cbor,
+            schema.tab_uuid_cbor,
+            schema.tab_oid_pickle,
+            schema.tab_str_pickle,
+            schema.tab_uuid_pickle,
+        ]
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True, stats=stats) as txn:
+                for tab in tabs:
+                    tab.truncate(txn)
+
+        logging.info(stats.puts)
+        logging.info(stats.dels)
+
+
+def test_fill_check(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema3()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.authid] = user
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                for user in testset1:
+                    _user = schema.users[txn, user.authid]
+                    assert _user
+                    assert _user == user
+
+
+def test_select(testset1):
+    testset1_keys = set([user.authid for user in testset1])
+
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema3()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.authid] = user
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                i = 0
+                for authid, user in schema.users.select(txn):
+                    i += 1
+                    assert user
+                    assert authid == user.authid
+                    assert authid in testset1_keys
+
+
+def test_count_all(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema3()
+
+        with zlmdb.Database(dbpath) as db:
+            # count on empty table
+            with db.begin() as txn:
+                cnt = schema.users.count(txn)
+                assert cnt == 0
+
+            # fill (and count on each insert)
+            with db.begin(write=True) as txn:
+                i = 0
+                for user in testset1:
+                    schema.users[txn, user.authid] = user
+                    i += 1
+
+                    # table count within filling transaction
+                    cnt = schema.users.count(txn)
+                    assert cnt == i
+
+                # table count within transaction
+                cnt = schema.users.count(txn)
+                assert cnt == len(testset1)
+
+            # table count in new transaction
+            with db.begin() as txn:
+                cnt = schema.users.count(txn)
+                assert cnt == len(testset1)
+
+        # table count in new connection
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                cnt = schema.users.count(txn)
+                assert cnt == len(testset1)
+
+
+def test_count_prefix(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema3()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.authid] = user
+
+        n = len(testset1)
+        tests = [
+            (None, n),
+            ("", n),
+            ("test-", n),
+            ("test-1", 111),
+            ("test-11", 11),
+            ("test-111", 1),
+        ]
+        with zlmdb.Database(dbpath) as db:
+            with db.begin() as txn:
+                for prefix, num in tests:
+                    cnt = schema.users.count(txn, prefix)
+                    assert cnt == num
+
+
+def test_fill_with_indexes(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema4()
+
+        with zlmdb.Database(dbpath) as db:
+            stats = zlmdb.TransactionStats()
+
+            with db.begin(write=True, stats=stats) as txn:
+                for user in testset1:
+                    schema.users[txn, user.oid] = user
+
+            # check indexes has been written to (in addition to the table itself)
+            num_indexes = len(schema.users.indexes())
+            assert stats.puts == len(testset1) * (1 + num_indexes)
+
+
+def test_truncate_table_with_index(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema4()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.oid] = user
+
+        stats = zlmdb.TransactionStats()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True, stats=stats) as txn:
+                records = schema.users.truncate(txn)
+                logging.info("table truncated: {}".format(records))
+
+        logging.info(stats.puts)
+        logging.info(stats.dels)
+
+
+def test_rebuild_index(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema4()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.oid] = user
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                records = schema.users.rebuild_index(txn, "idx1")
+                logging.info(
+                    '\nrebuilt specific index "idx1" on "users": {} records affected'.format(
+                        records
+                    )
+                )
+
+
+def test_rebuild_all_indexes(testset1):
+    with TemporaryDirectory() as dbpath:
+        logging.info("Using temporary directory {} for database".format(dbpath))
+
+        schema = Schema4()
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                for user in testset1:
+                    schema.users[txn, user.oid] = user
+
+        with zlmdb.Database(dbpath) as db:
+            with db.begin(write=True) as txn:
+                records = schema.users.rebuild_indexes(txn)
+                logging.info(
+                    '\nrebuilt all indexes on "users": {} records affected'.format(
+                        records
+                    )
+                )