rdbpython 0.2.1__cp313-cp313-macosx_11_0_arm64.whl

rdbpy/lib_utils.py

@@ -0,0 +1,52 @@
+import ctypes
+import os
+import platform
+from glob import glob
+
+
+def get_include_dir():
+    dirname = os.path.dirname(__file__)
+    abs_dirname = os.path.abspath(dirname)
+    path = os.path.join(abs_dirname, "include")
+    return path
+
+
+def get_lib_dir():
+    dirname = os.path.dirname(__file__)
+    abs_dirname = os.path.abspath(dirname)
+    path = abs_dirname
+    return path
+
+
+def get_libs():
+    return [
+        "rocksdb",
+    ]
+
+
+def get_lib_filename(name, lib_dir):
+    if platform.system() == "Darwin":
+        pattern = f"lib{name}*.dylib"
+    else:
+        pattern = f"lib{name}*.so*"
+
+    files = glob(os.path.join(lib_dir, pattern))
+    assert len(files) == 1
+    return os.path.basename(files[0])
+
+
+def get_lib_filenames():
+    local_lib_dir = get_lib_dir()
+    return [get_lib_filename(name, local_lib_dir) for name in get_libs()]
+
+
+def load_lib(filename) -> None:
+    local_lib_dir = get_lib_dir()
+    path = os.path.join(local_lib_dir, filename)
+    print(f"Loading runtime library from {path}")
+    ctypes.CDLL(path)
+
+
+def load_libs() -> None:
+    for filename in get_lib_filenames():
+        load_lib(filename)

rdbpy/logger.pxd

@@ -0,0 +1,5 @@
+cdef extern from "rocksdb/env.h" namespace "rocksdb":
+    cdef cppclass Logger:
+        pass
+
+    void Log(Logger*, const char*, ...) nogil except+

rdbpy/memtablerep.pxd

@@ -0,0 +1,12 @@
+from libc.stdint cimport int32_t
+
+cdef extern from "rocksdb/memtablerep.h" namespace "rocksdb":
+    cdef cppclass MemTableRepFactory:
+        MemTableRepFactory()
+
+    cdef MemTableRepFactory* NewHashSkipListRepFactory(size_t, int32_t, int32_t)
+    cdef MemTableRepFactory* NewHashLinkListRepFactory(size_t)
+
+cdef extern from "rdb_include/memtable_factories.hpp" namespace "py_rocks":
+    cdef MemTableRepFactory* NewVectorRepFactory(size_t)
+    cdef MemTableRepFactory* NewSkipListFactory()

rdbpy/merge_operator.pxd

@@ -0,0 +1,46 @@
+from libcpp.string cimport string
+from libcpp cimport bool as cpp_bool
+from libcpp.deque cimport deque
+from rdbpy.slice_ cimport Slice
+from rdbpy.logger cimport Logger
+from rdbpy.std_memory cimport shared_ptr
+
+cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb":
+    cdef cppclass MergeOperator:
+        pass
+
+ctypedef cpp_bool (*merge_func)(
+    void*,
+    const Slice&,
+    const Slice*,
+    const Slice&,
+    string*,
+    Logger*)
+
+ctypedef cpp_bool (*full_merge_func)(
+    void* ctx,
+    const Slice& key,
+    const Slice* existing_value,
+    const deque[string]& operand_list,
+    string* new_value,
+    Logger* logger)
+
+ctypedef cpp_bool (*partial_merge_func)(
+    void* ctx,
+    const Slice& key,
+    const Slice& left_op,
+    const Slice& right_op,
+    string* new_value,
+    Logger* logger)
+
+cdef extern from "rdb_include/merge_operator_wrapper.hpp" namespace "py_rocks":
+    cdef cppclass AssociativeMergeOperatorWrapper:
+        AssociativeMergeOperatorWrapper(string, void*, merge_func) nogil except+
+
+    cdef cppclass MergeOperatorWrapper:
+        MergeOperatorWrapper(
+            string,
+            void*,
+            void*,
+            full_merge_func,
+            partial_merge_func) nogil except+

rdbpy/merge_operators.py

@@ -0,0 +1,25 @@
+import struct as py_struct
+
+from rdbpy.interfaces import AssociativeMergeOperator
+
+
+class UintAddOperator(AssociativeMergeOperator):
+    def merge(self, key, existing_value, value):
+        if existing_value:
+            s = py_struct.unpack("Q", existing_value)[0] + py_struct.unpack("Q", value)[0]
+            return (True, py_struct.pack("Q", s))
+        return (True, value)
+
+    def name(self) -> bytes:
+        return b"uint64add"
+
+
+class StringAppendOperator(AssociativeMergeOperator):
+    def merge(self, key, existing_value, value):
+        if existing_value:
+            s = existing_value + b"," + value
+            return (True, s)
+        return (True, value)
+
+    def name(self) -> bytes:
+        return b"StringAppendOperator"

rdbpy/options.pxd

@@ -0,0 +1,179 @@
+from libcpp cimport bool as cpp_bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libc.stdint cimport uint64_t
+from libc.stdint cimport uint32_t
+from rdbpy.std_memory cimport shared_ptr
+from rdbpy.comparator cimport Comparator
+from rdbpy.merge_operator cimport MergeOperator
+from rdbpy.logger cimport Logger
+from rdbpy.slice_ cimport Slice
+from rdbpy.snapshot cimport Snapshot
+from rdbpy.slice_transform cimport SliceTransform
+from rdbpy.table_factory cimport TableFactory
+from rdbpy.memtablerep cimport MemTableRepFactory
+from rdbpy.universal_compaction cimport CompactionOptionsUniversal
+from rdbpy.cache cimport Cache
+
+cdef extern from "rocksdb/options.h" namespace "rocksdb":
+    cdef cppclass CompressionOptions:
+        int window_bits;
+        int level;
+        int strategy;
+        uint32_t max_dict_bytes
+        CompressionOptions() except +
+        CompressionOptions(int, int, int, int) except +
+
+    ctypedef enum CompactionStyle:
+        kCompactionStyleLevel
+        kCompactionStyleUniversal
+        kCompactionStyleFIFO
+        kCompactionStyleNone
+
+    ctypedef enum CompressionType:
+        kNoCompression
+        kSnappyCompression
+        kZlibCompression
+        kBZip2Compression
+        kLZ4Compression
+        kLZ4HCCompression
+        kXpressCompression
+        kZSTD
+        kZSTDNotFinalCompression
+        kDisableCompressionOption
+
+    ctypedef enum ReadTier:
+        kReadAllTier
+        kBlockCacheTier
+
+    ctypedef enum CompactionPri:
+        kByCompensatedSize
+        kOldestLargestSeqFirst
+        kOldestSmallestSeqFirst
+        kMinOverlappingRatio
+
+    # This needs to be in _rocksdb.pxd so it will export into python
+    #cpdef enum AccessHint "rocksdb::DBOptions::AccessHint":
+    #    NONE,
+    #    NORMAL,
+    #    SEQUENTIAL,
+    #    WILLNEED
+
+    cdef cppclass DBOptions:
+        cpp_bool create_if_missing
+        cpp_bool create_missing_column_families
+        cpp_bool error_if_exists
+        cpp_bool paranoid_checks
+        # TODO: env
+        shared_ptr[Logger] info_log
+        int max_open_files
+        int max_file_opening_threads
+        # TODO: statistics
+        cpp_bool use_fsync
+        string db_log_dir
+        string wal_dir
+        uint64_t delete_obsolete_files_period_micros
+        int max_background_jobs
+        int max_background_compactions
+        uint32_t max_subcompactions
+        int max_background_flushes
+        size_t max_log_file_size
+        size_t log_file_time_to_roll
+        size_t keep_log_file_num
+        size_t recycle_log_file_num
+        uint64_t max_manifest_file_size
+        int table_cache_numshardbits
+        uint64_t WAL_ttl_seconds
+        uint64_t WAL_size_limit_MB
+        size_t manifest_preallocation_size
+        cpp_bool allow_mmap_reads
+        cpp_bool allow_mmap_writes
+        cpp_bool use_direct_reads
+        cpp_bool use_direct_io_for_flush_and_compaction
+        cpp_bool allow_fallocate
+        cpp_bool is_fd_close_on_exec
+        unsigned int stats_dump_period_sec
+        cpp_bool advise_random_on_open
+        size_t db_write_buffer_size
+        # AccessHint access_hint_on_compaction_start
+        cpp_bool use_adaptive_mutex
+        uint64_t bytes_per_sync
+        cpp_bool allow_concurrent_memtable_write
+        cpp_bool enable_write_thread_adaptive_yield
+        shared_ptr[Cache] row_cache
+
+        cpp_bool skip_checking_sst_file_sizes_on_db_open
+        cpp_bool skip_stats_update_on_db_open
+
+    cdef cppclass ColumnFamilyOptions:
+        ColumnFamilyOptions()
+        ColumnFamilyOptions(const Options& options)
+        const Comparator* comparator
+        shared_ptr[MergeOperator] merge_operator
+        # TODO: compaction_filter
+        # TODO: compaction_filter_factory
+        size_t write_buffer_size
+        int max_write_buffer_number
+        int min_write_buffer_number_to_merge
+        CompressionType compression
+        CompactionPri compaction_pri
+        # TODO: compression_per_level
+        shared_ptr[SliceTransform] prefix_extractor
+        int num_levels
+        int level0_file_num_compaction_trigger
+        int level0_slowdown_writes_trigger
+        int level0_stop_writes_trigger
+        int max_mem_compaction_level
+        uint64_t target_file_size_base
+        int target_file_size_multiplier
+        uint64_t max_bytes_for_level_base
+        double max_bytes_for_level_multiplier
+        vector[int] max_bytes_for_level_multiplier_additional
+        int expanded_compaction_factor
+        int source_compaction_factor
+        int max_grandparent_overlap_factor
+        cpp_bool disableDataSync
+        size_t arena_block_size
+        # TODO: PrepareForBulkLoad()
+        cpp_bool disable_auto_compactions
+        cpp_bool allow_os_buffer
+        cpp_bool verify_checksums_in_compaction
+        CompactionStyle compaction_style
+        CompactionOptionsUniversal compaction_options_universal
+        cpp_bool filter_deletes
+        uint64_t max_sequential_skip_in_iterations
+        shared_ptr[MemTableRepFactory] memtable_factory
+        shared_ptr[TableFactory] table_factory
+        # TODO: table_properties_collectors
+        cpp_bool inplace_update_support
+        size_t inplace_update_num_locks
+        # TODO: remove options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor from document
+        uint64_t max_compaction_bytes
+        CompressionOptions compression_opts
+
+    cdef cppclass Options(DBOptions, ColumnFamilyOptions):
+        pass
+
+    cdef cppclass WriteOptions:
+        cpp_bool sync
+        cpp_bool disableWAL
+
+    cdef cppclass ReadOptions:
+        cpp_bool verify_checksums
+        cpp_bool fill_cache
+        const Snapshot* snapshot
+        ReadTier read_tier
+
+    cdef cppclass FlushOptions:
+        cpp_bool wait
+
+    ctypedef enum BottommostLevelCompaction:
+        blc_skip "rocksdb::BottommostLevelCompaction::kSkip"
+        blc_is_filter "rocksdb::BottommostLevelCompaction::kIfHaveCompactionFilter"
+        blc_force "rocksdb::BottommostLevelCompaction::kForce"
+
+    cdef cppclass CompactRangeOptions:
+        cpp_bool change_level
+        int target_level
+        uint32_t target_path_id
+        BottommostLevelCompaction bottommost_level_compaction

rdbpy/slice_.pxd

@@ -0,0 +1,20 @@
+from libcpp.string cimport string
+from libcpp cimport bool as cpp_bool
+
+cdef extern from "rocksdb/slice.h" namespace "rocksdb":
+    cdef cppclass Slice:
+        Slice() nogil
+        Slice(const char*, size_t) nogil
+        Slice(const string&) nogil
+        Slice(const char*) nogil
+
+        const char* data() nogil
+        size_t size() nogil
+        cpp_bool empty() nogil
+        char operator[](int) nogil
+        void clear() nogil
+        void remove_prefix(size_t) nogil
+        string ToString() nogil
+        string ToString(cpp_bool) nogil
+        int compare(const Slice&) nogil
+        cpp_bool starts_with(const Slice&) nogil

rdbpy/slice_transform.pxd

@@ -0,0 +1,37 @@
+from rdbpy.slice_ cimport Slice
+from libcpp.string cimport string
+from libcpp cimport bool as cpp_bool
+from rdbpy.logger cimport Logger
+from rdbpy.std_memory cimport shared_ptr
+
+cdef extern from "rocksdb/slice_transform.h" namespace "rocksdb":
+    cdef cppclass SliceTransform:
+        pass
+
+ctypedef Slice (*transform_func)(
+    void*,
+    Logger*,
+    string&,
+    const Slice&)
+
+ctypedef cpp_bool (*in_domain_func)(
+    void*,
+    Logger*,
+    string&,
+    const Slice&)
+
+ctypedef cpp_bool (*in_range_func)(
+    void*,
+    Logger*,
+    string&,
+    const Slice&)
+
+cdef extern from "rdb_include/slice_transform_wrapper.hpp" namespace "py_rocks":
+    cdef cppclass SliceTransformWrapper:
+        SliceTransformWrapper(
+                string name,
+                void*,
+                transform_func,
+                in_domain_func,
+                in_range_func) nogil except+
+        void set_info_log(shared_ptr[Logger]) nogil except+

rdbpy/snapshot.pxd

@@ -0,0 +1,3 @@
+cdef extern from "rocksdb/db.h" namespace "rocksdb":
+    cdef cppclass Snapshot:
+        pass

rdbpy/status.pxd

@@ -0,0 +1,15 @@
+from libcpp cimport bool as cpp_bool
+from libcpp.string cimport string
+
+cdef extern from "rocksdb/status.h" namespace "rocksdb":
+    cdef cppclass Status:
+        Status()
+        cpp_bool ok() nogil
+        cpp_bool IsNotFound() nogil const
+        cpp_bool IsCorruption() nogil const
+        cpp_bool IsNotSupported() nogil const
+        cpp_bool IsInvalidArgument() nogil const
+        cpp_bool IsIOError() nogil const
+        cpp_bool IsMergeInProgress() nogil const
+        cpp_bool IsIncomplete() nogil const
+        string ToString() nogil except+

rdbpy/std_memory.pxd

@@ -0,0 +1,7 @@
+cdef extern from "<memory>" namespace "std":
+    cdef cppclass shared_ptr[T]:
+        shared_ptr() nogil except+
+        shared_ptr(T*) nogil except+
+        void reset() nogil except+
+        void reset(T*) nogil except+
+        T* get() nogil except+

rdbpy/table_factory.pxd

@@ -0,0 +1,50 @@
+from libc.stdint cimport uint32_t
+from libcpp cimport bool as cpp_bool
+from rdbpy.std_memory cimport shared_ptr
+
+from rdbpy.cache cimport Cache
+from rdbpy.filter_policy cimport FilterPolicy
+
+cdef extern from "rocksdb/table.h" namespace "rocksdb":
+    cdef cppclass TableFactory:
+        TableFactory()
+
+    ctypedef enum BlockBasedTableIndexType:
+        kBinarySearch "rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch"
+        kHashSearch "rocksdb::BlockBasedTableOptions::IndexType::kHashSearch"
+
+    ctypedef enum ChecksumType:
+        kCRC32c
+        kxxHash
+
+    cdef cppclass BlockBasedTableOptions:
+        BlockBasedTableOptions()
+        BlockBasedTableIndexType index_type
+        cpp_bool hash_index_allow_collision
+        ChecksumType checksum
+        cpp_bool no_block_cache
+        size_t block_size
+        int block_size_deviation
+        int block_restart_interval
+        cpp_bool whole_key_filtering
+        shared_ptr[Cache] block_cache
+        shared_ptr[Cache] block_cache_compressed
+        shared_ptr[FilterPolicy] filter_policy
+
+    cdef TableFactory* NewBlockBasedTableFactory(const BlockBasedTableOptions&)
+
+    ctypedef enum EncodingType:
+        kPlain
+        kPrefix
+
+    cdef cppclass PlainTableOptions:
+        uint32_t user_key_len
+        int bloom_bits_per_key
+        double hash_table_ratio
+        size_t index_sparseness
+        size_t huge_page_tlb_size
+        EncodingType encoding_type
+        cpp_bool full_scan_mode
+        cpp_bool store_index_in_file
+
+    cdef TableFactory* NewPlainTableFactory(const PlainTableOptions&)

rdbpy/transaction.pxd

@@ -0,0 +1,154 @@
+from libc.stddef cimport size_t
+from libc.stdint cimport int64_t, uint32_t, uint64_t
+from libcpp cimport bool as cpp_bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+from rdbpy.slice_ cimport Slice
+from rdbpy.snapshot cimport Snapshot
+from rdbpy.status cimport Status
+from rdbpy.std_memory cimport shared_ptr
+
+cimport rdbpy.db as db
+cimport rdbpy.iterator as iterator
+cimport rdbpy.options as options
+
+
+cdef extern from "rocksdb/utilities/transaction_db_mutex.h" namespace "rocksdb":
+    cdef cppclass TransactionDBMutexFactory:
+        pass
+
+
+cdef extern from "rocksdb/utilities/transaction_db.h" namespace "rocksdb":
+    ctypedef uint64_t TransactionID
+    ctypedef string TransactionName
+
+    cdef enum TxnDBWritePolicy:
+        WRITE_COMMITTED
+        WRITE_PREPARED
+        WRITE_UNPREPARED
+
+    cdef cppclass LockManager:
+        pass
+
+    cdef cppclass LockManagerHandle:
+        LockManager* getLockManager() nogil except+
+
+    cdef cppclass TransactionDBOptions:
+        TransactionDBOptions() nogil except+
+        int64_t max_num_locks
+        uint32_t max_num_deadlocks
+        size_t num_stripes
+        int64_t transaction_lock_timeout
+        int64_t default_lock_timeout
+        shared_ptr[TransactionDBMutexFactory] custom_mutex_factory
+        TxnDBWritePolicy write_policy
+        cpp_bool rollback_merge_operands
+        shared_ptr[LockManagerHandle] lock_mgr_handle
+        cpp_bool skip_concurrency_control
+        int64_t default_write_batch_flush_threshold
+
+    cdef cppclass TransactionDBWriteOptimizations:
+        TransactionDBWriteOptimizations() nogil except+
+        cpp_bool skip_concurrency_control
+        cpp_bool skip_duplicate_key_check
+
+    cdef cppclass TransactionOptions:
+        TransactionOptions() nogil except+
+        cpp_bool set_snapshot
+        cpp_bool deadlock_detect
+        cpp_bool use_only_the_last_commit_time_batch_for_recovery
+        int64_t lock_timeout
+        int64_t expiration
+        int64_t deadlock_detect_depth
+        size_t max_write_batch_size
+        cpp_bool skip_concurrency_control
+        cpp_bool skip_prepare
+        int64_t write_batch_flush_threshold
+
+    cdef cppclass Transaction:
+        Status Commit() nogil except+
+        Status Rollback() nogil except+
+        Status Prepare() nogil except+
+        void SetSnapshot() nogil except+
+        const Snapshot* GetSnapshot() nogil except+
+        void ClearSnapshot() nogil except+
+        Status SetName(const TransactionName&) nogil except+
+        TransactionName GetName() nogil except+
+        TransactionID GetID() nogil except+
+        void SetSavePoint() nogil except+
+        Status PopSavePoint() nogil except+
+        Status RollbackToSavePoint() nogil except+
+        void DisableIndexing() nogil except+
+        void EnableIndexing() nogil except+
+        Status Put(db.ColumnFamilyHandle*, const Slice&, const Slice&, cpp_bool assume_tracked) nogil except+
+        Status Put(const Slice&, const Slice&) nogil except+
+        Status Merge(db.ColumnFamilyHandle*, const Slice&, const Slice&, cpp_bool assume_tracked) nogil except+
+        Status Merge(const Slice&, const Slice&) nogil except+
+        Status Delete(db.ColumnFamilyHandle*, const Slice&, cpp_bool assume_tracked) nogil except+
+        Status Delete(const Slice&) nogil except+
+        Status SingleDelete(db.ColumnFamilyHandle*, const Slice&, cpp_bool assume_tracked) nogil except+
+        Status SingleDelete(const Slice&) nogil except+
+        Status Get(const options.ReadOptions&, db.ColumnFamilyHandle*, const Slice&, string*) nogil except+
+        Status Get(const options.ReadOptions&, const Slice&, string*) nogil except+
+        vector[Status] MultiGet(
+            const options.ReadOptions&,
+            const vector[db.ColumnFamilyHandle*]&,
+            const vector[Slice]&,
+            vector[string]*) nogil except+
+        vector[Status] MultiGet(
+            const options.ReadOptions&,
+            const vector[Slice]&,
+            vector[string]*) nogil except+
+        iterator.Iterator* GetIterator(const options.ReadOptions&) nogil except+
+        iterator.Iterator* GetIterator(const options.ReadOptions&, db.ColumnFamilyHandle*) nogil except+
+
+    cdef cppclass CppTransactionDB "rocksdb::TransactionDB"(db.DB):
+        Transaction* BeginTransaction(
+            const options.WriteOptions&,
+            const TransactionOptions&,
+            Transaction* old_txn) nogil except+
+        Transaction* BeginTransaction(
+            const options.WriteOptions&,
+            const TransactionOptions&) nogil except+
+        Transaction* GetTransactionByName(const TransactionName&) nogil except+
+        void GetAllPreparedTransactions(vector[Transaction*]*) nogil except+
+
+    cdef Status TransactionDB_Open "rocksdb::TransactionDB::Open"(
+        const options.Options&,
+        const TransactionDBOptions&,
+        const string&,
+        CppTransactionDB**) nogil except+
+
+    cdef Status TransactionDB_Open_CF "rocksdb::TransactionDB::Open"(
+        const options.DBOptions&,
+        const TransactionDBOptions&,
+        const string&,
+        const vector[db.ColumnFamilyDescriptor]&,
+        vector[db.ColumnFamilyHandle*]*,
+        CppTransactionDB**) nogil except+
+
+
+# Helper functions to disambiguate overloaded Transaction methods
+cdef extern from "rocksdb/utilities/transaction.h":
+    """
+    static inline rocksdb::Iterator* Transaction_GetIterator(
+        rocksdb::Transaction* txn,
+        const rocksdb::ReadOptions& opts) {
+        return txn->GetIterator(opts);
+    }
+
+    static inline rocksdb::Iterator* Transaction_GetIterator_CF(
+        rocksdb::Transaction* txn,
+        const rocksdb::ReadOptions& opts,
+        rocksdb::ColumnFamilyHandle* cf) {
+        return txn->GetIterator(opts, cf);
+    }
+    """
+    iterator.Iterator* Transaction_GetIterator(
+        Transaction*,
+        const options.ReadOptions&) nogil except+
+    iterator.Iterator* Transaction_GetIterator_CF(
+        Transaction*,
+        const options.ReadOptions&,
+        db.ColumnFamilyHandle*) nogil except+

rdbpy/universal_compaction.pxd

@@ -0,0 +1,15 @@
+cdef extern from "rocksdb/universal_compaction.h" namespace "rocksdb":
+
+    ctypedef enum CompactionStopStyle:
+        kCompactionStopStyleSimilarSize
+        kCompactionStopStyleTotalSize
+
+    cdef cppclass CompactionOptionsUniversal:
+        CompactionOptionsUniversal()
+
+        unsigned int size_ratio
+        unsigned int min_merge_width
+        unsigned int max_merge_width
+        unsigned int max_size_amplification_percent
+        int compression_size_percent
+        CompactionStopStyle stop_style