virtuals-py 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/PKG-INFO +3 -3
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/README.md +2 -2
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/pyproject.toml +1 -1
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/observers/_base.py +4 -90
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/observers/mem.py +3 -16
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/observers/redis_pubsub.py +15 -38
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/publishers/_base.py +14 -68
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/publishers/mem.py +2 -5
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/publishers/redis_pubsub.py +5 -34
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/_deliver_local.py +2 -3
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/subscription.py +2 -3
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/types.py +2 -4
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/transport/transport.py +5 -32
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals_py.egg-info/PKG-INFO +3 -3
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/LICENSE.md +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/setup.cfg +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/codecs/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/codecs/json.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/codecs/msgpack.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/codecs/passthrough.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/codecs/pickle.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/binary/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/binary_py/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/binary_py/binary_codec.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/exceptions.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/string/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/string/string_codec.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/key_codecs/types.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/notify/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/notify/keyenc.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/notify/nu_notify.lua +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/notify/router.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/observers/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/publishers/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/_empty.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/_write_slot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/_writer.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/context.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/scan.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/snapshot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/storage.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/transaction.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/lmdb/write_batch.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/context.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/scan.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/snapshot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/sorted_keys.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/storage.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/store.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/transaction.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/write_batch.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/mem/write_slot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/context.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/scan.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/snapshot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/storage.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/transaction.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/rocksdb/write_batch.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/context.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/scan.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/snapshot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/storage.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/transaction.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/write_batch.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_backends/storages/sqlite/write_slot.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/_member_key.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/bytearray_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/dict_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/flat_dict_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/flat_list_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/fractional_index.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/fractional_indexed_dict_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/frozenset_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/indexed_dict_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/kh57_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/light_dict_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/list_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/log_indexed_dict_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/set_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/_views/tuple_view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/codecs/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/codecs/json.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/codecs/msgpack.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/codecs/passthrough.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/codecs/pickle.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/collections/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/collections/bases.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/collections/mapping.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/collections/protocols.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/collections/sequence.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/collections/set.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/container.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/container_ops.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/context.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/exceptions.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/marker.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/meta_ops.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/node_ops.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/types.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/container/validation_ops.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/constants.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/key.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/key_def.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/key_nav.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/path.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/path_def.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/path_nav.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/site.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/site_def.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/loc/site_nav.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/navigator.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/observers/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/observers/mem.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/observers/redis_pubsub.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/publishers/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/publishers/mem.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/publishers/redis_pubsub.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/storages/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/storages/lmdb.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/storages/mem.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/storages/rocksdb.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/storages/sqlite.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/testing/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/testing/codec_compliance.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/testing/observer_compliance.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/testing/storage_compliance.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/codec/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/codec/codec.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/codec/protocol.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/filter/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/filter/filter.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/filter/hints.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/exceptions.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/observer.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/observer/registry.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/publisher/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/publisher/exceptions.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/publisher/publisher.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/storage/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/storage/context.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/storage/exceptions.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/storage/scan.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/storage/storage.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/storage/types.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/transport/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/types/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/types/empty.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/types/key.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/types/observer.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/tkv/types/value.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/types/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/types/empty.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/types/not_set.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/types/tkv.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/base.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/bases.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/bases_observable.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/exceptions.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/registry.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/view/view.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals/views/__init__.py +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals_py.egg-info/SOURCES.txt +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals_py.egg-info/dependency_links.txt +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals_py.egg-info/requires.txt +0 -0
- {virtuals_py-0.2.0 → virtuals_py-0.2.1}/src/virtuals_py.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: virtuals-py
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Virtual Python collections over any storage.
|
|
5
5
|
Author-email: Gor Arakelyan <gorarkln@gmail.com>
|
|
6
6
|
Maintainer-email: Gor Arakelyan <gorarkln@gmail.com>
|
|
@@ -112,7 +112,7 @@ Swap `InMemoryStorage` for `SQLiteStorage`, `LMDBStorage` or `RocksDBStorage` an
|
|
|
112
112
|
- **Many structures, one store.** Dict, list, set, frozenset, tuple, bytearray, insertion-ordered dict with paging, fractional-indexed ordered collection with O(1) insert and move, int-keyed map with range sampling. Mix them freely under one root.
|
|
113
113
|
- **Real transactions.** Serializable, read your own writes, atomic commit, abort on exception. Snapshots bind at open and are never torn.
|
|
114
114
|
- **Concurrency that holds up.** One writer at a time, queued across threads, coroutines and processes. Coroutines wait without blocking the loop. Contention surfaces as two retryable errors, never as backend noise.
|
|
115
|
-
- **Watch anything.** Subscribe to a view, a child, its children or a pattern of descendants. Wakes arrive after commit, in-process or across processes over Redis or Valkey
|
|
115
|
+
- **Watch anything.** Subscribe to a view, a child, its children or a pattern of descendants. Wakes arrive after commit, in-process or across processes over Redis or Valkey. Delivery is best effort and drops are never silent.
|
|
116
116
|
- **Extensible both ways.** Write a new view and it runs on every backend. Write a new backend, run the shipped compliance suites, and every view works on it.
|
|
117
117
|
|
|
118
118
|
## Eager and lazy
|
|
@@ -192,7 +192,7 @@ observer.disconnect()
|
|
|
192
192
|
live.close()
|
|
193
193
|
```
|
|
194
194
|
|
|
195
|
-
Swap the in-memory pair for the Redis publisher and observer and the same watch fires in every process. Receivers re-read state on wake, so repeats
|
|
195
|
+
Swap the in-memory pair for the Redis publisher and observer and the same watch fires in every process. Receivers re-read state on wake, so repeats are harmless.
|
|
196
196
|
|
|
197
197
|
## Backends
|
|
198
198
|
|
|
@@ -59,7 +59,7 @@ Swap `InMemoryStorage` for `SQLiteStorage`, `LMDBStorage` or `RocksDBStorage` an
|
|
|
59
59
|
- **Many structures, one store.** Dict, list, set, frozenset, tuple, bytearray, insertion-ordered dict with paging, fractional-indexed ordered collection with O(1) insert and move, int-keyed map with range sampling. Mix them freely under one root.
|
|
60
60
|
- **Real transactions.** Serializable, read your own writes, atomic commit, abort on exception. Snapshots bind at open and are never torn.
|
|
61
61
|
- **Concurrency that holds up.** One writer at a time, queued across threads, coroutines and processes. Coroutines wait without blocking the loop. Contention surfaces as two retryable errors, never as backend noise.
|
|
62
|
-
- **Watch anything.** Subscribe to a view, a child, its children or a pattern of descendants. Wakes arrive after commit, in-process or across processes over Redis or Valkey
|
|
62
|
+
- **Watch anything.** Subscribe to a view, a child, its children or a pattern of descendants. Wakes arrive after commit, in-process or across processes over Redis or Valkey. Delivery is best effort and drops are never silent.
|
|
63
63
|
- **Extensible both ways.** Write a new view and it runs on every backend. Write a new backend, run the shipped compliance suites, and every view works on it.
|
|
64
64
|
|
|
65
65
|
## Eager and lazy
|
|
@@ -139,7 +139,7 @@ observer.disconnect()
|
|
|
139
139
|
live.close()
|
|
140
140
|
```
|
|
141
141
|
|
|
142
|
-
Swap the in-memory pair for the Redis publisher and observer and the same watch fires in every process. Receivers re-read state on wake, so repeats
|
|
142
|
+
Swap the in-memory pair for the Redis publisher and observer and the same watch fires in every process. Receivers re-read state on wake, so repeats are harmless.
|
|
143
143
|
|
|
144
144
|
## Backends
|
|
145
145
|
|
|
@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
|
|
|
10
10
|
# ============================================================================
|
|
11
11
|
[project]
|
|
12
12
|
name = "virtuals-py"
|
|
13
|
-
version = "0.2.
|
|
13
|
+
version = "0.2.1"
|
|
14
14
|
description = "Virtual Python collections over any storage."
|
|
15
15
|
readme = "README.md"
|
|
16
16
|
license = "Apache-2.0"
|
|
@@ -22,14 +22,12 @@ only for as long as that one callback. The worker holds the observer only
|
|
|
22
22
|
weakly between batches, so an observer dropped without disconnect() is
|
|
23
23
|
collected and its worker exits. An explicit disconnect() drops the wakes
|
|
24
24
|
still queued and closes every subscription (none survives it), so a reader
|
|
25
|
-
that reconnects subscribes again and re-reads
|
|
26
|
-
the contract is about a transport reconnect inside a live observer.
|
|
25
|
+
that reconnects subscribes again and re-reads.
|
|
27
26
|
|
|
28
27
|
Memory. A batch is at most `DISPATCH_BATCH_MAX` keys, so what one observer
|
|
29
28
|
holds is bounded by its dispatch queue, the batch in hand, and whatever the
|
|
30
29
|
transport parks on its own side (the Redis listener's overflow, at most one
|
|
31
|
-
more queue).
|
|
32
|
-
until it is delivered, however many publishers or reconnects ask for one.
|
|
30
|
+
more queue).
|
|
33
31
|
"""
|
|
34
32
|
|
|
35
33
|
from __future__ import annotations
|
|
@@ -60,21 +58,6 @@ if TYPE_CHECKING:
|
|
|
60
58
|
from virtuals.tkv.types import Key
|
|
61
59
|
|
|
62
60
|
|
|
63
|
-
def _hash_of(sub: Subscription) -> str:
|
|
64
|
-
"""``sub.filter_hash``, or a process-local stand-in for a filter without ``to_dict()``.
|
|
65
|
-
|
|
66
|
-
Such a filter cannot be subscribed on redis (its hash is sent at
|
|
67
|
-
subscribe), but on the in-memory transport it delivers keys, so its
|
|
68
|
-
resync wake is keyed by the subscription itself. The stand-in never
|
|
69
|
-
reaches a transport, and one subscription that cannot be hashed never
|
|
70
|
-
costs the others their wake.
|
|
71
|
-
"""
|
|
72
|
-
try:
|
|
73
|
-
return sub.filter_hash
|
|
74
|
-
except Exception: # no to_dict(), or one that does not serialise
|
|
75
|
-
return f"local:{id(sub):x}"
|
|
76
|
-
|
|
77
|
-
|
|
78
61
|
@dataclass(frozen=True, slots=True)
|
|
79
62
|
class _TaggedKey:
|
|
80
63
|
"""A key tagged with a filter hash for hash-scoped delivery."""
|
|
@@ -83,13 +66,6 @@ class _TaggedKey:
|
|
|
83
66
|
filter_hash: str
|
|
84
67
|
|
|
85
68
|
|
|
86
|
-
@dataclass(frozen=True, slots=True)
|
|
87
|
-
class _Resync:
|
|
88
|
-
"""A keyless resync wake for every local subscription on one filter hash."""
|
|
89
|
-
|
|
90
|
-
filter_hash: str
|
|
91
|
-
|
|
92
|
-
|
|
93
69
|
logger = getLogger(__name__)
|
|
94
70
|
|
|
95
71
|
|
|
@@ -203,10 +179,6 @@ class ObserverBase:
|
|
|
203
179
|
self._stop_event = threading.Event()
|
|
204
180
|
self._dispatch_thread: threading.Thread | None = None
|
|
205
181
|
self._worker_finalizer: weakref.finalize | None = None
|
|
206
|
-
# Filter hashes with a keyless resync wake queued (or parked by the
|
|
207
|
-
# transport) and not yet delivered: one wake per hash at a time.
|
|
208
|
-
self._resync_queued: set[str] = set()
|
|
209
|
-
self._resync_lock = threading.Lock()
|
|
210
182
|
|
|
211
183
|
def _ensure_connected(self) -> None:
|
|
212
184
|
if not self._connected:
|
|
@@ -231,8 +203,6 @@ class ObserverBase:
|
|
|
231
203
|
self._registry = SubscriptionRegistry()
|
|
232
204
|
stop = self._stop_event = threading.Event()
|
|
233
205
|
q = self._dispatch_queue = queue.Queue(maxsize=self._dispatch_queue_maxsize)
|
|
234
|
-
with self._resync_lock:
|
|
235
|
-
self._resync_queued = set()
|
|
236
206
|
|
|
237
207
|
self._on_connect()
|
|
238
208
|
|
|
@@ -381,48 +351,6 @@ class ObserverBase:
|
|
|
381
351
|
if not self._enqueue(k if filter_hash is None else _TaggedKey(k, filter_hash)):
|
|
382
352
|
return
|
|
383
353
|
|
|
384
|
-
def _dispatch_resync(self, filter_hash: str) -> None:
|
|
385
|
-
"""Queue one keyless resync wake for every local sub on `filter_hash`.
|
|
386
|
-
|
|
387
|
-
A transport calls this when it may have missed changes for that hash
|
|
388
|
-
(after a reconnect, once the hash is routed again). Each matching
|
|
389
|
-
subscription's receivers get `None` in place of a key, meaning
|
|
390
|
-
"re-read the state you watch". Queued behind keys already received,
|
|
391
|
-
so the wake follows them.
|
|
392
|
-
"""
|
|
393
|
-
if self._connected:
|
|
394
|
-
item = self._resync_item(filter_hash)
|
|
395
|
-
if item is not None:
|
|
396
|
-
self._enqueue(item)
|
|
397
|
-
|
|
398
|
-
def _dispatch_resync_all(self) -> None:
|
|
399
|
-
"""Queue one keyless resync wake per filter hash with a local subscription.
|
|
400
|
-
|
|
401
|
-
A transport calls this when a publisher on it dropped keys: nobody
|
|
402
|
-
knows which filters they matched, so every subscription re-reads.
|
|
403
|
-
"""
|
|
404
|
-
registry = self._registry
|
|
405
|
-
if not self._connected or registry is None:
|
|
406
|
-
return
|
|
407
|
-
for h in {_hash_of(s) for s in registry.subscriptions()}:
|
|
408
|
-
item = self._resync_item(h)
|
|
409
|
-
if item is not None:
|
|
410
|
-
self._enqueue(item)
|
|
411
|
-
|
|
412
|
-
def _resync_item(self, filter_hash: str) -> _Resync | None:
|
|
413
|
-
"""A resync wake to queue for `filter_hash`, or None while one is pending.
|
|
414
|
-
|
|
415
|
-
One keyless wake per hash is enough until it is delivered: it
|
|
416
|
-
re-reads state as of its delivery, which is later than every drop
|
|
417
|
-
or reconnect that asked for one in the meantime. So W publishers
|
|
418
|
-
dropping in one stall cost each subscription one re-read, not W.
|
|
419
|
-
"""
|
|
420
|
-
with self._resync_lock:
|
|
421
|
-
if filter_hash in self._resync_queued:
|
|
422
|
-
return None
|
|
423
|
-
self._resync_queued.add(filter_hash)
|
|
424
|
-
return _Resync(filter_hash)
|
|
425
|
-
|
|
426
354
|
def _enqueue_nowait(self, item: object) -> bool:
|
|
427
355
|
"""Put `item` on the dispatch queue if there is room. False when full."""
|
|
428
356
|
try:
|
|
@@ -470,30 +398,16 @@ class ObserverBase:
|
|
|
470
398
|
return
|
|
471
399
|
done.wait(timeout=max(0.0, deadline - time.monotonic()))
|
|
472
400
|
|
|
473
|
-
def _deliver_batch(self, batch: list[Key | _TaggedKey
|
|
401
|
+
def _deliver_batch(self, batch: list[Key | _TaggedKey]) -> None:
|
|
474
402
|
if not batch:
|
|
475
403
|
return
|
|
476
|
-
# Every resync wake in hand is consumed here, delivered or not, so
|
|
477
|
-
# its hash is released first: a later subscription on the same
|
|
478
|
-
# filter gets its own wake. Released before any receiver runs, so a
|
|
479
|
-
# drop recorded from now on queues a new wake.
|
|
480
|
-
with self._resync_lock:
|
|
481
|
-
for item in batch:
|
|
482
|
-
if isinstance(item, _Resync):
|
|
483
|
-
self._resync_queued.discard(item.filter_hash)
|
|
484
404
|
registry = self._registry
|
|
485
405
|
if registry is None:
|
|
486
406
|
return
|
|
487
407
|
if len(registry) == 0:
|
|
488
408
|
return
|
|
489
|
-
notifications: list[tuple[Key
|
|
409
|
+
notifications: list[tuple[Key, list[Subscription]]] = []
|
|
490
410
|
for item in batch:
|
|
491
|
-
if isinstance(item, _Resync):
|
|
492
|
-
h = item.filter_hash
|
|
493
|
-
woken = [s for s in registry.subscriptions() if _hash_of(s) == h]
|
|
494
|
-
if woken:
|
|
495
|
-
notifications.append((None, woken))
|
|
496
|
-
continue
|
|
497
411
|
if isinstance(item, _TaggedKey):
|
|
498
412
|
key = item.key
|
|
499
413
|
h = item.filter_hash
|
|
@@ -42,15 +42,6 @@ def _weak_listener(ref: weakref.ref[InMemoryObserver]) -> Callable[[list[Key]],
|
|
|
42
42
|
return listener
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
def _weak_resync(ref: weakref.ref[InMemoryObserver]) -> Callable[[], None]:
|
|
46
|
-
def on_resync() -> None:
|
|
47
|
-
obs = ref()
|
|
48
|
-
if obs is not None:
|
|
49
|
-
obs._dispatch_resync_all()
|
|
50
|
-
|
|
51
|
-
return on_resync
|
|
52
|
-
|
|
53
|
-
|
|
54
45
|
class InMemoryObserver(ObserverBase):
|
|
55
46
|
"""Observer that receives batches from a shared InMemoryTransport."""
|
|
56
47
|
|
|
@@ -63,16 +54,12 @@ class InMemoryObserver(ObserverBase):
|
|
|
63
54
|
super().__init__(dispatch_queue_maxsize=dispatch_queue_maxsize)
|
|
64
55
|
self._transport = transport
|
|
65
56
|
self._listener: Callable[[list[Key]], None] | None = None
|
|
66
|
-
self._on_resync: Callable[[], None] | None = None
|
|
67
57
|
|
|
68
58
|
def _on_connect(self) -> None:
|
|
69
|
-
|
|
70
|
-
self._listener
|
|
71
|
-
self._on_resync = _weak_resync(ref)
|
|
72
|
-
self._transport.register(self._listener, self._on_resync)
|
|
59
|
+
self._listener = _weak_listener(weakref.ref(self))
|
|
60
|
+
self._transport.register(self._listener)
|
|
73
61
|
|
|
74
62
|
def _on_disconnect(self) -> None:
|
|
75
63
|
if self._listener is not None:
|
|
76
|
-
self._transport.unregister(self._listener
|
|
64
|
+
self._transport.unregister(self._listener)
|
|
77
65
|
self._listener = None
|
|
78
|
-
self._on_resync = None
|
|
@@ -21,12 +21,10 @@ subscribe() in that window waits for the re-registration (or raises on
|
|
|
21
21
|
timeout) instead of returning early. Writes committed while the connection
|
|
22
22
|
is down, or before the re-registration lands, are NOT delivered (a restarted
|
|
23
23
|
server lost the index; a dropped connection misses the PUBLISH). The observer
|
|
24
|
-
logs a WARNING when this starts and an INFO per re-registered hash.
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
skip it. A publisher that had to drop keys sends the same keyless wake to
|
|
29
|
-
every hash on the prefix (an empty-keys payload) once it can publish again.
|
|
24
|
+
logs a WARNING when this starts and an INFO per re-registered hash. Delivery
|
|
25
|
+
is best effort: what was written in the window is gone, and notifications
|
|
26
|
+
flow again once the hash is re-registered. Level flows recover on their own
|
|
27
|
+
re-check; event flows miss those changes.
|
|
30
28
|
|
|
31
29
|
Backlog. The listener never blocks on the dispatch queue. When the queue is
|
|
32
30
|
full it parks inbound keys in its own overflow buffer (bounded by the same
|
|
@@ -40,10 +38,7 @@ a SUBSCRIBE awaits its confirmation: then it reads whatever the backlog, so
|
|
|
40
38
|
the confirmation is never behind it and a subscribe() never times out on a
|
|
41
39
|
slow receiver. While one is pending the overflow is unbounded; the real cap
|
|
42
40
|
is valkey's client output buffer limit, past which valkey disconnects the
|
|
43
|
-
client, which degrades to a reconnect and
|
|
44
|
-
(reconnect or a publisher's drop) is parked at most once per hash until
|
|
45
|
-
delivered, so W publishers dropping in one stall cost one re-read per
|
|
46
|
-
subscription, not W.
|
|
41
|
+
client, which degrades to a reconnect (and the reconnect window above).
|
|
47
42
|
|
|
48
43
|
Timeouts. Both connections have a socket and connect timeout and at most one
|
|
49
44
|
transparent redis-py retry, so a stalled server bounds `nu1_reg` (and with it
|
|
@@ -64,8 +59,8 @@ messages whose `iid` matches.
|
|
|
64
59
|
|
|
65
60
|
Wire format:
|
|
66
61
|
- Channel: `{prefix}:notif:{filter_hash}`.
|
|
67
|
-
- msgpack payload: `{"iid": pub_id, "keys": [[...], ...]}`.
|
|
68
|
-
|
|
62
|
+
- msgpack payload: `{"iid": pub_id, "keys": [[...], ...]}`. A payload
|
|
63
|
+
without keys is ignored.
|
|
69
64
|
- Routing index: `{prefix}:nx1:*` (owned by the Lua library).
|
|
70
65
|
Mixing with observers/publishers from before server-side routing on one
|
|
71
66
|
prefix is unsupported.
|
|
@@ -88,7 +83,7 @@ from virtuals._backends.notify.router import NotifyLua
|
|
|
88
83
|
from virtuals.tkv.filter import Filter, canonicalize, filter_hash
|
|
89
84
|
from virtuals.tkv.observer import ObserverSubscriptionError
|
|
90
85
|
|
|
91
|
-
from ._base import DEFAULT_DISPATCH_QUEUE_MAXSIZE, ObserverBase,
|
|
86
|
+
from ._base import DEFAULT_DISPATCH_QUEUE_MAXSIZE, ObserverBase, _TaggedKey
|
|
92
87
|
|
|
93
88
|
|
|
94
89
|
try:
|
|
@@ -351,7 +346,7 @@ class RedisObserver(ObserverBase):
|
|
|
351
346
|
self._pending: dict[bytes, deque[list[_Ack]]] = {}
|
|
352
347
|
self._watched_conn: Any = None
|
|
353
348
|
# Inbound items the dispatch queue had no room for (listener-owned).
|
|
354
|
-
self._overflow: deque[_TaggedKey
|
|
349
|
+
self._overflow: deque[_TaggedKey] = deque()
|
|
355
350
|
|
|
356
351
|
self._obs_stop_event = threading.Event()
|
|
357
352
|
self._listener_thread: threading.Thread | None = None
|
|
@@ -712,8 +707,7 @@ class RedisObserver(ObserverBase):
|
|
|
712
707
|
if reset:
|
|
713
708
|
logger.warning(
|
|
714
709
|
"Redis observer pubsub connection %s: %d subscribed filter hash(es) not "
|
|
715
|
-
"routed until re-registered; writes in this window are not delivered,
|
|
716
|
-
"subscribers get a keyless resync wake once re-registered",
|
|
710
|
+
"routed until re-registered; writes in this window are not delivered",
|
|
717
711
|
why,
|
|
718
712
|
reset,
|
|
719
713
|
)
|
|
@@ -733,12 +727,6 @@ class RedisObserver(ObserverBase):
|
|
|
733
727
|
ack = None
|
|
734
728
|
elif action == "unsub":
|
|
735
729
|
self._pubsub.unsubscribe(channel)
|
|
736
|
-
elif action == "resync":
|
|
737
|
-
# Behind every key parked so far, so the wake follows them.
|
|
738
|
-
item = self._resync_item(channel)
|
|
739
|
-
if item is not None:
|
|
740
|
-
self._overflow.append(item)
|
|
741
|
-
self._drain_overflow()
|
|
742
730
|
except Exception as e:
|
|
743
731
|
logger.warning("Listener cmd %s(%s) failed: %s", action, channel, e)
|
|
744
732
|
if ack is not None:
|
|
@@ -853,7 +841,7 @@ class RedisObserver(ObserverBase):
|
|
|
853
841
|
# callback queue room. In that state the overflow can absorb what
|
|
854
842
|
# the server buffered for this client, so the real bound is valkey's
|
|
855
843
|
# pubsub output buffer limit; past it valkey disconnects the client,
|
|
856
|
-
# which degrades to a reconnect
|
|
844
|
+
# which degrades to a reconnect.
|
|
857
845
|
room = len(self._overflow) < self._dispatch_queue_maxsize or pending
|
|
858
846
|
|
|
859
847
|
# 3. Take one message if redis-py has one and there is room for it,
|
|
@@ -900,18 +888,11 @@ class RedisObserver(ObserverBase):
|
|
|
900
888
|
# multiple registered filters (arriving on multiple notif channels),
|
|
901
889
|
# and drops the router's over-delivery (hints are a superset).
|
|
902
890
|
raw_keys = payload.get("keys")
|
|
903
|
-
if raw_keys:
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
# A publisher that dropped keys asks every hash to re-read. One
|
|
907
|
-
# wake per hash is pending at a time, whatever the publisher count.
|
|
908
|
-
item = self._resync_item(filter_hash_str)
|
|
909
|
-
if item is None:
|
|
910
|
-
return
|
|
911
|
-
self._overflow.append(item)
|
|
912
|
-
else:
|
|
913
|
-
logger.warning("Payload without keys on %s ignored", msg.get("channel"))
|
|
891
|
+
if not raw_keys:
|
|
892
|
+
if raw_keys != []:
|
|
893
|
+
logger.warning("Payload without keys on %s ignored", msg.get("channel"))
|
|
914
894
|
return
|
|
895
|
+
self._overflow.extend(_TaggedKey(tuple(k), filter_hash_str) for k in raw_keys)
|
|
915
896
|
self._drain_overflow()
|
|
916
897
|
|
|
917
898
|
# -- Maintenance thread ---------------------------------------------------
|
|
@@ -929,10 +910,6 @@ class RedisObserver(ObserverBase):
|
|
|
929
910
|
return False
|
|
930
911
|
logger.info("Re-registered filter hash %s after reconnect", h)
|
|
931
912
|
self._mark_ready(h, gen)
|
|
932
|
-
# Routed again: wake each local sub once so it re-reads whatever was
|
|
933
|
-
# written while the hash was not routed. Through the listener, so
|
|
934
|
-
# the wake queues behind keys it read before the reconnect.
|
|
935
|
-
self._dispatch_pubsub_cmd("resync", h, wait=False)
|
|
936
913
|
return True
|
|
937
914
|
|
|
938
915
|
def _mark_ready(self, h: str, gen: int) -> None:
|
|
@@ -54,7 +54,7 @@ DEFAULT_QUEUE_MAXSIZE: int = 100_000
|
|
|
54
54
|
DEFAULT_PUBLISH_RETRY_SECONDS: float = 5.0
|
|
55
55
|
|
|
56
56
|
# Longest notify() blocks a writer on a full queue before dropping the rest
|
|
57
|
-
# of its keys (counted
|
|
57
|
+
# of its keys (counted and surfaced, never delivered).
|
|
58
58
|
DEFAULT_NOTIFY_TIMEOUT_SECONDS: float = 1.0
|
|
59
59
|
|
|
60
60
|
# Retry backoff: first delay, doubling, capped.
|
|
@@ -68,7 +68,7 @@ _STOP_MARKER: object = object()
|
|
|
68
68
|
# drain up to it.
|
|
69
69
|
_STOP_JOIN_SECONDS: float = 5.0
|
|
70
70
|
|
|
71
|
-
# Idle poll of the worker
|
|
71
|
+
# Idle poll of the worker.
|
|
72
72
|
_IDLE_POLL_SECONDS: float = 0.1
|
|
73
73
|
|
|
74
74
|
|
|
@@ -94,11 +94,8 @@ def _worker(ref: weakref.ref[PublisherBase], q: queue.Queue[Any], stop: threadin
|
|
|
94
94
|
except queue.Empty:
|
|
95
95
|
if stop.is_set():
|
|
96
96
|
return
|
|
97
|
-
|
|
98
|
-
if owner is None:
|
|
97
|
+
if ref() is None:
|
|
99
98
|
return
|
|
100
|
-
owner._try_resync()
|
|
101
|
-
del owner
|
|
102
99
|
continue
|
|
103
100
|
|
|
104
101
|
# Drain what's queued right now into a single batch. Sentinels
|
|
@@ -106,10 +103,7 @@ def _worker(ref: weakref.ref[PublisherBase], q: queue.Queue[Any], stop: threadin
|
|
|
106
103
|
batch: list[Key] = []
|
|
107
104
|
while True:
|
|
108
105
|
if item is _STOP_MARKER:
|
|
109
|
-
|
|
110
|
-
owner = ref()
|
|
111
|
-
if owner is not None and not owner._drain_failed:
|
|
112
|
-
owner._try_resync(force=True) # a drop just before stop still wakes
|
|
106
|
+
_publish(ref, batch, stop)
|
|
113
107
|
return
|
|
114
108
|
if isinstance(item, threading.Event):
|
|
115
109
|
alive = _publish(ref, batch, stop)
|
|
@@ -154,10 +148,8 @@ class PublisherBase:
|
|
|
154
148
|
`publish_retry_seconds`; meanwhile the worker does not drain, the queue
|
|
155
149
|
fills and notify() blocks (backpressure) for up to `notify_timeout`,
|
|
156
150
|
then drops the rest of its keys. Every drop is logged at ERROR, counted
|
|
157
|
-
in `dropped_keys
|
|
158
|
-
|
|
159
|
-
subscriber on the prefix to re-read (`_send_resync`, transport
|
|
160
|
-
specific), so a level flow recovers what the drop lost.
|
|
151
|
+
in `dropped_keys` and raised once by the next flush(). Delivery is best
|
|
152
|
+
effort: dropped keys never reach a subscriber.
|
|
161
153
|
|
|
162
154
|
disconnect() drains: the worker publishes every key queued before it
|
|
163
155
|
(one attempt per chunk, no retry window; after the first failed chunk
|
|
@@ -191,13 +183,9 @@ class PublisherBase:
|
|
|
191
183
|
self.dropped_keys: int = 0
|
|
192
184
|
self._last_error: BaseException | None = None
|
|
193
185
|
self._error_lock = threading.Lock()
|
|
194
|
-
# Set by any drop; cleared once a resync wake went out (worker only).
|
|
195
|
-
self._resync_pending: bool = False
|
|
196
|
-
self._next_resync_at: float = 0.0
|
|
197
|
-
self._resync_backoff: float = _RETRY_BACKOFF_START
|
|
198
186
|
# Set by a notify() drop, cleared by the worker's next successful
|
|
199
187
|
# send: while set, writers drop at once instead of each waiting out
|
|
200
|
-
# `notify_timeout` (the
|
|
188
|
+
# `notify_timeout` (the transport is stalled, so the wait buys nothing).
|
|
201
189
|
self._stalled: bool = False
|
|
202
190
|
# Set once a chunk failed after stop: the rest of the drain is dropped.
|
|
203
191
|
self._drain_failed: bool = False
|
|
@@ -232,7 +220,7 @@ class PublisherBase:
|
|
|
232
220
|
q = self._queue = queue.Queue(maxsize=self._queue_maxsize)
|
|
233
221
|
self._drain_failed = False
|
|
234
222
|
# Per connection: a fresh queue has room, so writers wait for it
|
|
235
|
-
# again.
|
|
223
|
+
# again.
|
|
236
224
|
self._stalled = False
|
|
237
225
|
|
|
238
226
|
self._on_connect()
|
|
@@ -327,8 +315,8 @@ class PublisherBase:
|
|
|
327
315
|
Fire-and-forget: caller just enqueues; the worker handles batching
|
|
328
316
|
and transport send. Blocks on a full queue for at most
|
|
329
317
|
`notify_timeout` in total, then drops the keys it could not queue
|
|
330
|
-
(logged, counted, surfaced by flush()
|
|
331
|
-
|
|
318
|
+
(logged, counted, surfaced by flush()): a committing writer is
|
|
319
|
+
never held by a stalled transport. Once a call
|
|
332
320
|
has dropped, later calls on the still-full queue drop at once until
|
|
333
321
|
the worker sends again, so a stall costs the first commit the
|
|
334
322
|
timeout and the rest nothing. With `notify_timeout=None` the call
|
|
@@ -337,8 +325,7 @@ class PublisherBase:
|
|
|
337
325
|
Accepts a single key (tuple) or a batch of keys (set, list, etc.).
|
|
338
326
|
|
|
339
327
|
While not connected the keys cannot be sent: they are counted as a
|
|
340
|
-
drop (logged, surfaced by the next flush()
|
|
341
|
-
wake once a connection sends again) and `PublisherConnectionError`
|
|
328
|
+
drop (logged, surfaced by the next flush()) and `PublisherConnectionError`
|
|
342
329
|
is raised. A commit that lands between disconnect() and connect()
|
|
343
330
|
is therefore never lost silently.
|
|
344
331
|
"""
|
|
@@ -405,13 +392,11 @@ class PublisherBase:
|
|
|
405
392
|
done.wait(timeout=max(0.0, deadline - time.monotonic()))
|
|
406
393
|
self._raise_last_error()
|
|
407
394
|
|
|
408
|
-
def _record_drop(self, n: int, err: BaseException
|
|
409
|
-
"""Count a drop and surface it.
|
|
395
|
+
def _record_drop(self, n: int, err: BaseException) -> None:
|
|
396
|
+
"""Count a drop and surface it."""
|
|
410
397
|
with self._error_lock:
|
|
411
398
|
self.dropped_keys += n
|
|
412
399
|
self._last_error = err
|
|
413
|
-
if resync:
|
|
414
|
-
self._resync_pending = True
|
|
415
400
|
|
|
416
401
|
def _raise_last_error(self) -> None:
|
|
417
402
|
with self._error_lock:
|
|
@@ -439,7 +424,7 @@ class PublisherBase:
|
|
|
439
424
|
prepared, n = self._prepare_chunk(chunk)
|
|
440
425
|
except Exception as e:
|
|
441
426
|
logger.error("Publish failed, dropping %d keys: %s", len(chunk), e)
|
|
442
|
-
self._record_drop(len(chunk), e
|
|
427
|
+
self._record_drop(len(chunk), e)
|
|
443
428
|
return
|
|
444
429
|
if not n:
|
|
445
430
|
return
|
|
@@ -467,38 +452,8 @@ class PublisherBase:
|
|
|
467
452
|
delay = min(delay * 2, _RETRY_BACKOFF_CAP)
|
|
468
453
|
continue
|
|
469
454
|
self._stalled = False # writers may wait for room again
|
|
470
|
-
if self._resync_pending:
|
|
471
|
-
self._try_resync()
|
|
472
455
|
return
|
|
473
456
|
|
|
474
|
-
def _try_resync(self, *, force: bool = False) -> None:
|
|
475
|
-
"""Send the pending resync wake if due. Worker only.
|
|
476
|
-
|
|
477
|
-
Called after every successful send and on every idle poll, so the
|
|
478
|
-
wake goes out as soon as the transport is back even with nothing
|
|
479
|
-
else to publish. Failures back off up to the retry cap (`force`
|
|
480
|
-
ignores the backoff: the last attempt before the worker exits).
|
|
481
|
-
|
|
482
|
-
The flag is cleared before the send and set back on failure: a drop
|
|
483
|
-
recorded while the wake is in flight keeps its own wake, since that
|
|
484
|
-
drop's commit may have landed after this wake was published.
|
|
485
|
-
"""
|
|
486
|
-
if not self._resync_pending:
|
|
487
|
-
return
|
|
488
|
-
if not force and time.monotonic() < self._next_resync_at:
|
|
489
|
-
return
|
|
490
|
-
self._resync_pending = False
|
|
491
|
-
try:
|
|
492
|
-
self._send_resync()
|
|
493
|
-
except Exception as e:
|
|
494
|
-
logger.warning("Publisher resync wake failed, retrying: %s", e)
|
|
495
|
-
self._resync_pending = True
|
|
496
|
-
self._next_resync_at = time.monotonic() + self._resync_backoff
|
|
497
|
-
self._resync_backoff = min(self._resync_backoff * 2, _RETRY_BACKOFF_CAP)
|
|
498
|
-
return
|
|
499
|
-
self._resync_backoff = _RETRY_BACKOFF_START
|
|
500
|
-
logger.info("Publisher sent a resync wake after dropping keys")
|
|
501
|
-
|
|
502
457
|
# -- Subclass hooks -------------------------------------------------------
|
|
503
458
|
|
|
504
459
|
def _on_connect(self) -> None:
|
|
@@ -525,15 +480,6 @@ class PublisherBase:
|
|
|
525
480
|
"""Send one prepared chunk; retried on error. Default: `_publish_batch`."""
|
|
526
481
|
self._publish_batch(prepared)
|
|
527
482
|
|
|
528
|
-
def _send_resync(self) -> None:
|
|
529
|
-
"""Ask every subscriber on this transport to re-read, after a drop.
|
|
530
|
-
|
|
531
|
-
Default: nothing (the drop stays counted and surfaced by flush()).
|
|
532
|
-
Transports that can reach their subscribers implement it; it is
|
|
533
|
-
retried until it succeeds.
|
|
534
|
-
"""
|
|
535
|
-
return None
|
|
536
|
-
|
|
537
483
|
def _publish_batch(self, batch: list[Key]) -> None:
|
|
538
484
|
"""Send a batch of keys onto the transport. Subclass responsibility."""
|
|
539
485
|
raise NotImplementedError
|
|
@@ -7,8 +7,8 @@ worker thread.
|
|
|
7
7
|
There is no stalled server to protect a writer against, so notify() keeps
|
|
8
8
|
pure backpressure by default (`notify_timeout=None`: it blocks until the
|
|
9
9
|
queue has room, never drops while connected; a notify while disconnected
|
|
10
|
-
is a counted drop like on any publisher). With a timeout set, a
|
|
11
|
-
|
|
10
|
+
is a counted drop like on any publisher). With a timeout set, a full queue
|
|
11
|
+
drops keys too: counted and logged, never delivered.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
@@ -48,9 +48,6 @@ class InMemoryPublisher(PublisherBase):
|
|
|
48
48
|
def _publish_batch(self, batch: list[Key]) -> None:
|
|
49
49
|
self._transport.publish(batch)
|
|
50
50
|
|
|
51
|
-
def _send_resync(self) -> None:
|
|
52
|
-
self._transport.resync()
|
|
53
|
-
|
|
54
51
|
|
|
55
52
|
if TYPE_CHECKING:
|
|
56
53
|
from virtuals.tkv.publisher import PublisherProtocol
|
|
@@ -14,14 +14,9 @@ opaque strings, and Python's Filter.matches stays the only exact matcher
|
|
|
14
14
|
Delivery is at least once per retry: a chunk retried after a lost reply runs
|
|
15
15
|
PUBLISH again, so an observer can see a key twice (see PublisherBase).
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
payload to every hash registered on the prefix: the observer turns that
|
|
21
|
-
into one keyless resync wake per local subscription, so level flows re-read
|
|
22
|
-
and recover what the drop lost. Every observer on the prefix is woken, not
|
|
23
|
-
only those whose filters the dropped keys matched: the keys are gone before
|
|
24
|
-
routing, and an extra re-read is harmless.
|
|
17
|
+
Delivery is best effort. A drop (retry window exhausted, notify() timed out
|
|
18
|
+
on a full queue, unroutable key, drain cut short) is logged, counted and
|
|
19
|
+
raised by the next flush(), but the dropped keys never reach an observer.
|
|
25
20
|
|
|
26
21
|
The connection has a socket and connect timeout and at most one transparent
|
|
27
22
|
redis-py retry, so one send attempt is bounded by about twice the socket
|
|
@@ -29,8 +24,7 @@ timeout; the worker's own retry window stays in charge of the deadline.
|
|
|
29
24
|
|
|
30
25
|
Wire format (unchanged for observers):
|
|
31
26
|
- Channel: `{prefix}:notif:{filter_hash}`.
|
|
32
|
-
- msgpack payload: `{"iid": pub_id, "keys": [[...], ...]}`.
|
|
33
|
-
is a keyless resync wake.
|
|
27
|
+
- msgpack payload: `{"iid": pub_id, "keys": [[...], ...]}`.
|
|
34
28
|
|
|
35
29
|
Mixing publishers/observers from before server-side routing on one prefix is
|
|
36
30
|
unsupported: the routing index lives under `{prefix}:nx1:*`.
|
|
@@ -93,9 +87,6 @@ MAX_KEYS_PER_CALL: int = 250
|
|
|
93
87
|
# send after this instead of hanging the worker.
|
|
94
88
|
DEFAULT_SOCKET_TIMEOUT_SECONDS: float = 2.0
|
|
95
89
|
|
|
96
|
-
# msgpack array header for zero keys: the resync payload is `hdr + this`.
|
|
97
|
-
_EMPTY_KEYS: bytes = b"\x90"
|
|
98
|
-
|
|
99
90
|
|
|
100
91
|
def _close_connection(conn: object) -> None:
|
|
101
92
|
"""Close the server connection. Runs from disconnect() and from the finalizer."""
|
|
@@ -190,9 +181,7 @@ class RedisPublisher(PublisherBase):
|
|
|
190
181
|
# Not a str/int key, an int outside msgpack's range, or a str
|
|
191
182
|
# with lone surrogates. Logged, counted, raised by next flush().
|
|
192
183
|
logger.error("Skipping unroutable key %r: %s", key, e)
|
|
193
|
-
|
|
194
|
-
# so no subscriber is behind on it: counted, no resync wake.
|
|
195
|
-
self._record_drop(1, e, resync=False)
|
|
184
|
+
self._record_drop(1, e)
|
|
196
185
|
continue
|
|
197
186
|
args.append(packed)
|
|
198
187
|
args.append(b"%d" % len(segs))
|
|
@@ -210,24 +199,6 @@ class RedisPublisher(PublisherBase):
|
|
|
210
199
|
raise redis.ConnectionError("RedisPublisher not connected")
|
|
211
200
|
self._lua.call(self._redis_pub, "nu1_route", prepared)
|
|
212
201
|
|
|
213
|
-
def _send_resync(self) -> None:
|
|
214
|
-
"""Publish an empty-keys payload to every hash registered on the prefix.
|
|
215
|
-
|
|
216
|
-
Two round trips, no Lua: the hashes come from the routing index's
|
|
217
|
-
refs hash, each gets one PUBLISH in a pipeline. A hash with no
|
|
218
|
-
subscriber left is unregistered lazily by the next routed publish.
|
|
219
|
-
"""
|
|
220
|
-
if self._redis_pub is None:
|
|
221
|
-
raise redis.ConnectionError("RedisPublisher not connected")
|
|
222
|
-
hashes = self._redis_pub.hkeys(f"{self._channel_prefix}:nx1:refs")
|
|
223
|
-
if not hashes:
|
|
224
|
-
return
|
|
225
|
-
payload = self._hdr + _EMPTY_KEYS
|
|
226
|
-
pipe = self._redis_pub.pipeline(transaction=False)
|
|
227
|
-
for h in hashes:
|
|
228
|
-
pipe.publish(b"%s:notif:%s" % (self._channel_prefix.encode(), h), payload)
|
|
229
|
-
pipe.execute()
|
|
230
|
-
|
|
231
202
|
def _publish_batch(self, batch: list[Key]) -> None:
|
|
232
203
|
"""Encode and send one chunk (no retry)."""
|
|
233
204
|
args, n = self._route_args(batch)
|
|
@@ -25,13 +25,12 @@ __all__ = [
|
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def deliver_local(
|
|
28
|
-
notifications: list[tuple[Key
|
|
28
|
+
notifications: list[tuple[Key, list[Subscription]]],
|
|
29
29
|
) -> None:
|
|
30
30
|
"""Deliver notifications to local subscribers.
|
|
31
31
|
|
|
32
32
|
Args:
|
|
33
|
-
notifications: Matched (key, subscriptions) pairs.
|
|
34
|
-
keyless resync wake.
|
|
33
|
+
notifications: Matched (key, subscriptions) pairs.
|
|
35
34
|
"""
|
|
36
35
|
for key, subs in notifications:
|
|
37
36
|
for sub in subs:
|