pyfusefilter 1.3.1__cp313-cp313-win_amd64.whl

pyfusefilter/__init__.py

@@ -0,0 +1,4 @@
+from pyfusefilter.pyfusefilter import Xor8, Xor16, Fuse8, Fuse16
+
+VERSION = "1.2.1"
+__all__ = ["Xor8", "Xor16", "Fuse8", "Fuse16"]

pyfusefilter/ffibuild.py

@@ -0,0 +1,175 @@
+from cffi import FFI
+import os
+from sys import exit, platform
+
+
+ffi = FFI()
+cdef_from_file = None
+
+all_src = ""
+with open("lib/xor_singleheader/include/xorfilter.h", "r", encoding='utf-8') as src:
+  all_src += src.read()
+
+with open("lib/xor_singleheader/include/binaryfusefilter.h", "r", encoding='utf-8') as src:
+  all_src += src.read()
+
+
+ffi.set_source(
+  "_xorfilter", all_src,
+)
+
+ffi.cdef(
+    """
+static inline uint64_t xor_murmur64(uint64_t h) ;
+
+static inline uint64_t xor_mix_split(uint64_t key, uint64_t seed) ;
+
+static inline uint64_t xor_rotl64(uint64_t n, unsigned int c) ;
+
+static inline uint32_t xor_reduce(uint32_t hash, uint32_t n) ;
+
+static inline uint64_t xor_fingerprint(uint64_t hash) ;
+
+static inline uint64_t xor_rng_splitmix64(uint64_t *seed) ;
+
+
+typedef struct xor8_s {
+  uint64_t seed;
+  uint64_t blockLength;
+  uint8_t
+      *fingerprints; // after xor8_allocate, will point to 3*blockLength values
+} xor8_t;
+
+struct xor_xorset_s {
+  uint64_t xormask;
+  uint32_t count;
+};
+
+typedef struct xor16_s {
+  uint64_t seed;
+  uint64_t blockLength;
+  uint16_t
+      *fingerprints; // after xor16_allocate, will point to 3*blockLength values
+} xor16_t;
+
+typedef struct binary_fuse8_s {
+  uint64_t Seed;
+  uint32_t Size;
+  uint32_t SegmentLength;
+  uint32_t SegmentLengthMask;
+  uint32_t SegmentCount;
+  uint32_t SegmentCountLength;
+  uint32_t ArrayLength;
+  uint8_t *Fingerprints;
+} binary_fuse8_t;
+
+typedef struct binary_fuse16_s {
+  uint64_t Seed;
+  uint32_t Size;
+  uint32_t SegmentLength;
+  uint32_t SegmentLengthMask;
+  uint32_t SegmentCount;
+  uint32_t SegmentCountLength;
+  uint32_t ArrayLength;
+  uint16_t *Fingerprints;
+} binary_fuse16_t;
+
+typedef struct xor_xorset_s xor_xorset_t;
+
+struct xor_hashes_s {
+  uint64_t h;
+  uint32_t h0;
+  uint32_t h1;
+  uint32_t h2;
+};
+typedef struct xor_hashes_s xor_hashes_t;
+
+struct xor_h0h1h2_s {
+  uint32_t h0;
+  uint32_t h1;
+  uint32_t h2;
+};
+typedef struct xor_h0h1h2_s xor_h0h1h2_t;
+
+struct xor_keyindex_s {
+  uint64_t hash;
+  uint32_t index;
+};
+
+typedef struct xor_keyindex_s xor_keyindex_t;
+
+struct xor_setbuffer_s {
+  xor_keyindex_t *buffer;
+  uint32_t *counts;
+  int insignificantbits;
+  uint32_t slotsize; // should be 1<< insignificantbits
+  uint32_t slotcount;
+  size_t originalsize;
+};
+typedef struct xor_setbuffer_s xor_setbuffer_t;
+
+static inline bool xor8_contain(uint64_t key, const xor8_t *filter);
+static inline bool xor16_contain(uint64_t key, const xor16_t *filter);
+static inline bool binary_fuse8_contain(uint64_t key, const binary_fuse8_t *filter);
+static inline bool binary_fuse16_contain(uint64_t key, const binary_fuse16_t *filter);
+
+static inline bool xor8_allocate(uint32_t size, xor8_t *filter);
+static inline bool xor16_allocate(uint32_t size, xor16_t *filter);
+static inline bool binary_fuse8_allocate(uint32_t size, binary_fuse8_t *filter);
+static inline bool binary_fuse16_allocate(uint32_t size, binary_fuse16_t *filter);
+
+static inline size_t xor8_size_in_bytes(const xor8_t *filter);
+static inline size_t xor16_size_in_bytes(const xor16_t *filter);
+static inline size_t binary_fuse8_size_in_bytes(const binary_fuse8_t *filter);
+static inline size_t binary_fuse16_size_in_bytes(const binary_fuse16_t *filter);
+
+static inline void xor8_free(xor8_t *filter);
+static inline void xor16_free(xor16_t *filter);
+static inline void binary_fuse8_free(binary_fuse8_t *filter);
+static inline void binary_fuse16_free(binary_fuse16_t *filter);
+
+static inline xor_hashes_t xor8_get_h0_h1_h2(uint64_t k, const xor8_t *filter) ;
+static inline uint32_t xor8_get_h0(uint64_t hash, const xor8_t *filter) ;
+static inline uint32_t xor8_get_h1(uint64_t hash, const xor8_t *filter) ;
+static inline uint32_t xor8_get_h2(uint64_t hash, const xor8_t *filter) ;
+static inline uint32_t xor16_get_h0(uint64_t hash, const xor16_t *filter) ;
+static inline uint32_t xor16_get_h1(uint64_t hash, const xor16_t *filter) ;
+static inline uint32_t xor16_get_h2(uint64_t hash, const xor16_t *filter) ;
+static inline xor_hashes_t xor16_get_h0_h1_h2(uint64_t k, const xor16_t *filter) ;
+
+static inline bool xor_init_buffer(xor_setbuffer_t *buffer, size_t size) ;
+static inline void xor_free_buffer(xor_setbuffer_t *buffer) ;
+static inline void xor_buffered_increment_counter(uint32_t index, uint64_t hash, xor_setbuffer_t *buffer, xor_xorset_t *sets) ;
+static inline void xor_make_buffer_current(xor_setbuffer_t *buffer, xor_xorset_t *sets, uint32_t index, xor_keyindex_t *Q, size_t *Qsize) ;
+static inline void xor_buffered_decrement_counter(uint32_t index, uint64_t hash, xor_setbuffer_t *buffer, xor_xorset_t *sets, xor_keyindex_t *Q, size_t *Qsize) ;
+static inline void xor_flush_increment_buffer(xor_setbuffer_t *buffer, xor_xorset_t *sets) ;
+static inline void xor_flush_decrement_buffer(xor_setbuffer_t *buffer, xor_xorset_t *sets, xor_keyindex_t *Q, size_t *Qsize) ;
+static inline uint32_t xor_flushone_decrement_buffer(xor_setbuffer_t *buffer, xor_xorset_t *sets, xor_keyindex_t *Q, size_t *Qsize) ;
+
+bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) ;
+bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) ;
+bool binary_fuse8_populate(const uint64_t *keys, uint32_t size, binary_fuse8_t *filter) ;
+bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) ;
+bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) ;
+bool binary_fuse16_populate(const uint64_t *keys, uint32_t size, binary_fuse16_t *filter) ;
+
+
+size_t xor16_serialization_bytes(xor16_t *filter);
+size_t xor8_serialization_bytes(const xor8_t *filter);
+void xor16_serialize(const xor16_t *filter, char *buffer);
+void xor8_serialize(const xor8_t *filter, char *buffer);
+bool xor16_deserialize(xor16_t * filter, const char *buffer);
+bool xor8_deserialize(xor8_t * filter, const char *buffer);
+
+size_t binary_fuse16_serialization_bytes(binary_fuse16_t *filter);
+size_t binary_fuse8_serialization_bytes(const binary_fuse8_t *filter);
+void binary_fuse16_serialize(const binary_fuse16_t *filter, char *buffer);
+void binary_fuse8_serialize(const binary_fuse8_t *filter, char *buffer);
+bool binary_fuse16_deserialize(binary_fuse16_t * filter, const char *buffer);
+bool binary_fuse8_deserialize(binary_fuse8_t * filter, const char *buffer);
+
+"""
+)
+
+if __name__ == "__main__":
+    ffi.compile(verbose=True)

pyfusefilter/pyfusefilter.py

@@ -0,0 +1,412 @@
+from ._xorfilter import lib, ffi
+from ctypes import c_ulonglong
+import xxhash
+import struct
+
+def hash(item):
+    # xxhash 4 no longer accepts str; encoding as UTF-8 gives exactly the
+    # same hash values that earlier xxhash versions computed for str input.
+    return xxhash.xxh64_intdigest(str(item).encode("utf-8"))
+
+class Xor8:
+    """
+    Xor8 is a probabilistic data structure that allows for fast set membership queries.
+    It may return false positives but never false negatives. It uses slightly more than
+    a byte of memory per key. Its false positive rate is roughly 0.39%.
+    For large sets, a Fuse8 filter is more efficient than a Xor8 filter.
+    """
+    def __init__(self, size_or_data):
+        """
+        Initialize the Xor8 filter.
+        If an integer is provided, allocate a filter of that size. You must then use 'populate' to add data.
+        If an iterable is provided, populate the filter with the data.
+        """
+        self.__filter = ffi.new("xor8_t *")
+        if isinstance(size_or_data, int):
+            status = lib.xor8_allocate(size_or_data, self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+        else:
+            data = list(size_or_data)
+            status = lib.xor8_allocate(len(data), self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+            else:
+                data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+                if not lib.xor8_buffered_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
+
+    def __repr__(self):
+        return "Xor8 object with size(in bytes):{}".format(self.size_in_bytes())
+
+    def __getitem__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __contains__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __del__(self):
+        lib.xor8_free(self.__filter)
+
+    def populate(self, data: list):
+        """
+        Set the data of the filter. You may use this method to add data after
+        allocating the filter with an integer size. The sizes should match.
+        You can reuse a filter with new data (i.e., call populate several times)
+        as long as the size remains a constant.
+        """
+        data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+        return lib.xor8_buffered_populate(data, len(data), self.__filter)
+
+    def contains(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        item = c_ulonglong((hash(item))).value
+        return lib.xor8_contain(item, self.__filter)
+
+    def size_in_bytes(self):
+        """
+        Return the size of the filter in bytes, not counting Python overhead.
+        This value might be slightly larger than the serialized size.
+        """
+        return lib.xor8_size_in_bytes(self.__filter)
+    
+    def serialize(self):
+        """
+        Serialize the filter to a bytes object.
+        """
+        buffer = ffi.new("char[]", lib.xor8_serialization_bytes(self.__filter))
+        lib.xor8_serialize(self.__filter, buffer)
+        return ffi.buffer(buffer)
+
+    @staticmethod
+    def deserialize(buffer):
+        """
+        Deserialize a bytes object to a Xor8 filter.
+        """
+        self = object.__new__(Xor8)
+        self.__filter = ffi.new("xor8_t *")
+        lib.xor8_deserialize(self.__filter, ffi.from_buffer(buffer))
+        return self
+
+
+class Xor16:
+    """
+    Xor16 is a probabilistic data structure that allows for fast set membership queries.
+    It may return false positives but never false negatives. It uses about two bytes of memory per key.
+    Its false positive rate is roughly 0.0015%.
+    For large sets, a Fuse16 filter is more efficient than a Xor16 filter.
+    """
+    def __init__(self, size_or_data):
+        """
+        Initialize the Xor16 filter.
+        If an integer is provided, allocate a filter of that size. You must then use 'populate' to add data.
+        If an iterable is provided, populate the filter with the data.
+        """
+        self.__filter = ffi.new("xor16_t *")
+        if isinstance(size_or_data, int):
+            status = lib.xor16_allocate(size_or_data, self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+        else:
+            data = list(size_or_data)
+            status = lib.xor16_allocate(len(data), self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+            else:
+                data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+                if not lib.xor16_buffered_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
+
+    def __repr__(self):
+        """
+        Return a string representation of the Xor16 filter, including its size in bytes.
+        """
+        return "Xor16 object with size(in bytes):{}".format(self.size_in_bytes())
+
+    def __getitem__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __contains__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __del__(self):
+        """
+        Free the memory allocated for the Xor16 filter.
+        """
+        lib.xor16_free(self.__filter)
+
+    def populate(self, data):
+        """
+        Set the data of the filter. You may use this method to add data after
+        allocating the filter with an integer size. The sizes should match.
+        You can reuse a filter with new data (i.e., call populate several times)
+        as long as the size remains a constant.
+        """
+        data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+        return lib.xor16_buffered_populate(data, len(data), self.__filter)
+
+    def contains(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        item = c_ulonglong((hash(item))).value
+        return lib.xor16_contain(item, self.__filter)
+
+    def size_in_bytes(self):
+        """
+        Return the size of the filter in bytes, not counting Python overhead.
+        This value might be slightly larger than the serialized size.
+        """
+        return lib.xor16_size_in_bytes(self.__filter)
+
+    def serialize(self):
+        """
+        Serialize the filter to a bytes object.
+        """
+        buffer = ffi.new("char[]", lib.xor16_serialization_bytes(self.__filter))
+        lib.xor16_serialize(self.__filter, buffer)
+        return ffi.buffer(buffer)
+
+    @staticmethod
+    def deserialize(buffer):
+        """
+        Deserialize a bytes object to a Xor16 filter.
+        """
+        self = object.__new__(Xor16)
+        self.__filter = ffi.new("xor16_t *")
+        lib.xor16_deserialize(self.__filter, ffi.from_buffer(buffer))
+        return self
+
+class Fuse8:
+    """
+    Fuse8 is a probabilistic data structure for fast set membership queries.
+    It offers a balance between speed and memory usage, with a false positive rate of about  0.39%.
+    It uses about one byte of memory per key.
+    For large sets, a Fuse8 filter is more efficient than a Xor8 filter.
+    """
+    def __init__(self, size_or_data):
+        """
+        Initialize the Fuse8 filter.
+        If an integer is provided, allocate a filter of that size. You must then use 'populate' to add data.
+        If an iterable is provided, populate the filter with the data.
+        """
+        self.__filter = ffi.new("binary_fuse8_t *")
+        if isinstance(size_or_data, int):
+            status = lib.binary_fuse8_allocate(size_or_data, self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+        else:
+            data = list(size_or_data)
+            status = lib.binary_fuse8_allocate(len(data), self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+            else:
+                data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+                if not lib.binary_fuse8_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
+
+    def __repr__(self):
+        """
+        Return a string representation of the Fuse8 filter, including its size in bytes.
+        """
+        return "Fuse8 object with size(in bytes):{}".format(self.size_in_bytes())
+
+    def __getitem__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __contains__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __del__(self):
+        """
+        Free the memory allocated for the Fuse8 filter.
+        """
+        lib.binary_fuse8_free(self.__filter)
+
+    def populate(self, data: list):
+        """
+        Set the data of the filter. You may use this method to add data after
+        allocating the filter with an integer size. If the number of items
+        differs from the allocated size, the filter is reallocated.
+        You can reuse a filter with new data (i.e., call populate several times).
+        """
+        data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+        if len(data) != self.__filter.Size:
+            lib.binary_fuse8_free(self.__filter)
+            if not lib.binary_fuse8_allocate(len(data), self.__filter):
+                raise MemoryError("Unable to allocate memory for filter")
+        return lib.binary_fuse8_populate(data, len(data), self.__filter)
+
+    def contains(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        item = c_ulonglong((hash(item))).value
+        return lib.binary_fuse8_contain(item, self.__filter)
+
+    def size_in_bytes(self):
+        """
+        Return the size of the filter in bytes, not counting Python overhead.
+        This value might be slightly larger than the serialized size.
+        """
+        return lib.binary_fuse8_size_in_bytes(self.__filter)
+
+    def serialize(self):
+        """
+        Serialize the filter to a bytes object.
+        """
+        buffer = ffi.new("char[]", lib.binary_fuse8_serialization_bytes(self.__filter))
+        lib.binary_fuse8_serialize(self.__filter, buffer)
+        return ffi.buffer(buffer)
+
+    @staticmethod
+    def deserialize(buffer):
+        """
+        Deserialize a bytes object to a Fuse8 filter.
+        """
+        self = object.__new__(Fuse8)
+        self.__filter = ffi.new("binary_fuse8_t *")
+        lib.binary_fuse8_deserialize(self.__filter, ffi.from_buffer(buffer))
+        return self
+
+class Fuse16:
+    """
+    Fuse16 is a probabilistic data structure that allows for fast set membership queries.
+    It may return false positives but never false negatives. It uses about two bytes of memory per key.
+    Its false positive rate is roughly 0.0015%.
+    For large sets, a Fuse16 filter is more efficient than a Xor16 filter.
+    """
+    def __init__(self, size_or_data):
+        """
+        Initialize the Fuse16 filter.
+        If an integer is provided, allocate a filter of that size. You must then use 'populate' to add data.
+        If an iterable is provided, populate the filter with the data.
+        """
+        self.__filter = ffi.new("binary_fuse16_t *")
+        if isinstance(size_or_data, int):
+            status = lib.binary_fuse16_allocate(size_or_data, self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+        else:
+            data = list(size_or_data)
+            status = lib.binary_fuse16_allocate(len(data), self.__filter)
+            if not status:
+                raise MemoryError("Unable to allocate memory for filter")
+            else:
+                data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+                if not lib.binary_fuse16_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
+
+    def __repr__(self):
+        """
+        Return a string representation of the Fuse16 filter, including its size in bytes.
+        """
+        return "Fuse16 object with size(in bytes):{}".format(self.size_in_bytes())
+
+    def __getitem__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __contains__(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        return self.contains(item)
+
+    def __del__(self):
+        """
+        Free the memory allocated for the Fuse16 filter.
+        """
+        lib.binary_fuse16_free(self.__filter)
+
+    def populate(self, data: list):
+        """
+        Set the data of the filter. You may use this method to add data after
+        allocating the filter with an integer size. If the number of items
+        differs from the allocated size, the filter is reallocated.
+        You can reuse a filter with new data (i.e., call populate several times).
+        """
+        data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+        if len(data) != self.__filter.Size:
+            lib.binary_fuse16_free(self.__filter)
+            if not lib.binary_fuse16_allocate(len(data), self.__filter):
+                raise MemoryError("Unable to allocate memory for filter")
+        return lib.binary_fuse16_populate(data, len(data), self.__filter)
+
+    def contains(self, item):
+        """
+        Check if the item is in the filter. With a small probability, the filter
+        may return true for an item not in the set (false positive). There is
+        no false negative: if the filter returns false, the item is definitely not in the set.
+        """
+        item = c_ulonglong((hash(item))).value
+        return lib.binary_fuse16_contain(item, self.__filter)
+
+    def size_in_bytes(self):
+        """
+        Return the size of the filter in bytes, not counting Python overhead.
+        This value might be slightly larger than the serialized size.
+        """
+        return lib.binary_fuse16_size_in_bytes(self.__filter)
+
+    def serialize(self):
+        """
+        Serialize the filter to a bytes object.
+        """
+        buffer = ffi.new("char[]", lib.binary_fuse16_serialization_bytes(self.__filter))
+        lib.binary_fuse16_serialize(self.__filter, buffer)
+        return ffi.buffer(buffer)
+
+    @staticmethod
+    def deserialize(buffer):
+        """
+        Deserialize a bytes object to a Fuse16 filter.
+        """
+        self = object.__new__(Fuse16)
+        self.__filter = ffi.new("binary_fuse16_t *")
+        lib.binary_fuse16_deserialize(self.__filter, ffi.from_buffer(buffer))
+        return self