roaringbitmap 0.1__tar.gz → 0.3__tar.gz

{roaringbitmap-0.1 → roaringbitmap-0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: roaringbitmap
-Version: 0.1
+Version: 0.3
 Summary: Roaring Bitmap
 Home-page: https://github.com/andreasvc/roaringbitmap/
 Author: Andreas van Cranenburgh
@@ -47,6 +47,8 @@ Description: Roaring Bitmap in Cython
             >>> RoaringBitmap(range(10)) & RoaringBitmap(range(5, 15))
             RoaringBitmap({5, 6, 7, 8, 9})
         
+        For API documentation cf. https://pythonhosted.org/roaringbitmap/
+        
         Benchmarks
         ----------
         Output of ``$ python benchmarks.py``::
@@ -102,4 +104,3 @@ Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python :: 2.7
 Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Cython
-Requires: cython (>=0.20)

{roaringbitmap-0.1 → roaringbitmap-0.3}/README.rst

@@ -39,6 +39,8 @@ Python set containing (unsigned) 32-bit integers::
     >>> RoaringBitmap(range(10)) & RoaringBitmap(range(5, 15))
     RoaringBitmap({5, 6, 7, 8, 9})
 
+For API documentation cf. https://pythonhosted.org/roaringbitmap/
+
 Benchmarks
 ----------
 Output of ``$ python benchmarks.py``::

{roaringbitmap-0.1 → roaringbitmap-0.3}/setup.py

@@ -3,15 +3,20 @@ import os
 import sys
 from distutils.core import setup
 from distutils.extension import Extension
-try:
-	from Cython.Build import cythonize
-	from Cython.Distutils import build_ext
-except ImportError as err:
-	print('ERROR: Cython not found.')
-	sys.exit(1)
+USE_CYTHON = '--with-cython' in sys.argv
+if USE_CYTHON:
+	sys.argv.remove('--with-cython')
+	try:
+		from Cython.Build import cythonize
+		from Cython.Distutils import build_ext
+	except ImportError:
+		raise ValueError('could not import Cython.')
+	cmdclass = dict(build_ext=build_ext)
+else:
+	cmdclass = dict()
 
 metadata = dict(name='roaringbitmap',
-		version='0.1',
+		version='0.3',
 		description='Roaring Bitmap',
 		long_description=open('README.rst').read(),
 		author='Andreas van Cranenburgh',
@@ -26,9 +31,6 @@ metadata = dict(name='roaringbitmap',
 				'Programming Language :: Python :: 3.3',
 				'Programming Language :: Cython',
 		],
-		requires=[
-				'cython (>=0.20)',
-		],
 )
 
 # some of these directives increase performance,
@@ -51,19 +53,27 @@ directives = {
 
 if __name__ == '__main__':
 	os.environ['GCC_COLORS'] = 'auto'
-	extensions = [Extension(
-			'*',
-			sources=['src/*.pyx'],
-			extra_compile_args=['-O3', '-DNDEBUG', '-march=native'],
-			# extra_compile_args=['-O0', '-g'],
-			# extra_link_args=['-g'],
-			)]
+	if USE_CYTHON:
+		extensions = [Extension(
+				'*',
+				sources=['src/*.pyx'],
+				extra_compile_args=['-O3', '-DNDEBUG', '-march=native'],
+				# extra_compile_args=['-O0', '-g'],
+				# extra_link_args=['-g'],
+				)]
+		ext_modules = cythonize(
+				extensions,
+				annotate=True,
+				compiler_directives=directives,
+				language_level=3,
+		)
+	else:
+		ext_modules = [Extension(
+				'roaringbitmap',
+				sources=['src/roaringbitmap.c'],
+				extra_compile_args=['-O3', '-DNDEBUG', '-march=native'],
+				)]
 	setup(
-			cmdclass=dict(build_ext=build_ext),
-			ext_modules=cythonize(
-					extensions,
-					annotate=True,
-					compiler_directives=directives,
-					language_level=3,
-			),
+			cmdclass=cmdclass,
+			ext_modules=ext_modules,
 			**metadata)

{roaringbitmap-0.1 → roaringbitmap-0.3}/src/arrayops.pxi

@@ -100,9 +100,9 @@ cdef int advance(uint16_t *data, int pos, int length, uint16_t minitem):
 		return upper
 	if data[upper] < minitem:
 		return length
-	lower += spansize // 2
+	lower += spansize >> 1
 	while lower + 1 != upper:
-		mid = (lower + upper) // 2
+		mid = (<unsigned int>lower + <unsigned int>upper) >> 1
 		if data[mid] == minitem:
 			return mid
 		elif data[mid] < minitem:

{roaringbitmap-0.1 → roaringbitmap-0.3}/src/bitcount.h

@@ -27,7 +27,7 @@ extern "C" {
 
 #ifdef BITCOUNT_VS_X86
 #include <intrin.h>
-#pragma intrinsic(_BitScanForward,_BitScanReverse,__popcnt)
+#pragma intrinsic(_BitScanForward64,_BitScanReverse64,__popcnt64)
 #endif
 
 #include <limits.h>
@@ -43,10 +43,10 @@ unsigned int bit_popcount_general(uint64_t);
    bit position. If v is 0, the result is undefined. */
 BITCOUNT_INLINE unsigned int bit_clz(uint64_t v) {
 	#if defined(BITCOUNT_GCC)
-	return __builtin_clzl(v);
+	return __builtin_clzll(v);
 	#elif defined(BITCOUNT_VS_X86)
 	uint64_t result;
-	_BitScanReverse(&result, v);
+	_BitScanReverse64(&result, v);
 	return BITCOUNT_BITS - 1 - result;
 	#else
 	return bit_clz_general(v);
@@ -57,10 +57,10 @@ BITCOUNT_INLINE unsigned int bit_clz(uint64_t v) {
    bit position. If v is 0, the result is undefined. */
 BITCOUNT_INLINE unsigned int bit_ctz(uint64_t v) {
 	#if defined(BITCOUNT_GCC)
-	return __builtin_ctzl(v);
+	return __builtin_ctzll(v);
 	#elif defined(BITCOUNT_VS_X86)
 	uint64_t result;
-	_BitScanForward(&result, v);
+	_BitScanForward64(&result, v);
 	return result;
 	#else
 	return bit_ctz_general(v);
@@ -70,9 +70,9 @@ BITCOUNT_INLINE unsigned int bit_ctz(uint64_t v) {
 /* Returns the number of 1-bits in v. */
 BITCOUNT_INLINE unsigned int bit_popcount(uint64_t v) {
 	#if defined(BITCOUNT_GCC)
-	return __builtin_popcountl(v);
+	return __builtin_popcountll(v);
 	#elif defined(BITCOUNT_VS_X86)
-	return __popcnt(v);
+	return __popcnt64(v);
 	#else
 	return bit_popcount_general(v);
 	#endif

{roaringbitmap-0.1 → roaringbitmap-0.3}/src/bitops.pxi

@@ -44,7 +44,7 @@ cdef inline int iteratesetbits(uint64_t *vec, int slots,
 			return -1
 		cur[0] = vec[idx[0]]
 	tmp = bit_ctz(cur[0])  # index of right-most 1-bit in current slot
-	cur[0] ^= 1UL << tmp  # TOGGLEBIT(cur, tmp)
+	cur[0] ^= 1ULL << tmp  # TOGGLEBIT(cur, tmp)
 	return idx[0] * BITSIZE + tmp
 
 
@@ -60,7 +60,7 @@ cdef inline int iterateunsetbits(uint64_t *vec, int slots,
 			return -1
 		cur[0] = ~vec[idx[0]]
 	tmp = bit_ctz(cur[0])  # index of right-most 0-bit in current slot
-	cur[0] ^= 1UL << tmp  # TOGGLEBIT(cur, tmp)
+	cur[0] ^= 1ULL << tmp  # TOGGLEBIT(cur, tmp)
 	return idx[0] * BITSIZE + tmp
 
 
@@ -90,7 +90,7 @@ cdef inline int reviteratesetbits(uint64_t *vec, uint64_t *cur, int *idx):
 			return -1
 		cur[0] = vec[idx[0]]
 	tmp = BITSIZE - bit_clz(cur[0]) - 1  # index of left-most 1-bit in cur
-	cur[0] &= ~(1UL << tmp)  # CLEARBIT(cur, tmp)
+	cur[0] &= ~(1ULL << tmp)  # CLEARBIT(cur, tmp)
 	return idx[0] * BITSIZE + tmp
 
 

{roaringbitmap-0.1 → roaringbitmap-0.3}/src/block.pxi

@@ -3,7 +3,6 @@ cdef inline Block new_Block(uint16_t key):
 	block.key = key
 	block.state = POSITIVE
 	block.cardinality = 0
-	block.buf = array.clone(ushortarray, 0, False)
 	return block
 
 
@@ -17,7 +16,7 @@ cdef class Block(object):
 	cdef uint8_t state  # either DENSE, INVERTED, or POSITIVE
 	cdef uint16_t key  # the high bits of elements in this block
 	cdef uint32_t cardinality  # the number of elements
-	cdef array.array buf  # bitvector, positive sparse, or inverted sparse
+	cdef array.array buf  # data: sparse array or fixed-size bitvector
 
 	def __cinit__(self):
 		pass
@@ -86,6 +85,27 @@ cdef class Block(object):
 				del self.buf[i]
 				self.cardinality -= 1
 
+	cdef _and(self, Block other):
+		cdef Block result
+		cdef int n
+		if self.state == DENSE:
+			if other.state == POSITIVE:
+				result = new_Block(self.key)
+				result.buf = array.clone(ushortarray, other.cardinality, False)
+				result.state = POSITIVE
+				result.cardinality = 0
+				for n in range(other.cardinality):
+					if TESTBIT(self.buf.data.as_ulongs,
+							other.buf.data.as_ushorts[n]):
+						result.buf.data.as_ushorts[self.cardinality
+								] = other.buf.data.as_ushorts[n]
+						result.cardinality += 1
+				array.resize(result.buf, result.cardinality)
+				return result
+		result = self.copy()
+		result &= other
+		return result
+
 	cdef iand(self, Block other):
 		cdef array.array tmp
 		cdef int length, n
@@ -99,9 +119,9 @@ cdef class Block(object):
 						tmp.data.as_ushorts[self.cardinality
 								] = other.buf.data.as_ushorts[n]
 						self.cardinality += 1
-				array.resize(tmp, self.cardinality)
-				self.state = POSITIVE
 				self.buf = tmp
+				array.resize(self.buf, self.cardinality)
+				self.state = POSITIVE
 			elif other.state == DENSE:
 				self.cardinality = bitsetintersectinplace(
 						self.buf.data.as_ulongs,
@@ -114,22 +134,19 @@ cdef class Block(object):
 								other.buf.data.as_ushorts[n])
 						self.cardinality -= 1
 		elif other.state == DENSE:
+			tmp = array.clone(ushortarray, BITMAPSIZE // sizeof(short), False)
 			if self.state == INVERTED:
-				tmp = array.clone(ushortarray, BITMAPSIZE, False)
-				memset(tmp.data.as_uchars, 255, BLOCKSIZE // sizeof(char))
+				memset(tmp.data.as_chars, 255, BITMAPSIZE)
 				for n in range(BLOCKSIZE - self.cardinality):
-					CLEARBIT(tmp.data.as_ulongs,
-							self.buf.data.as_ushorts[n])
+					CLEARBIT(tmp.data.as_ulongs, self.buf.data.as_ushorts[n])
 			else:  # self.state == POSITIVE:
-				tmp = array.clone(ushortarray, BITMAPSIZE, True)
+				memset(tmp.data.as_chars, 0, BITMAPSIZE)
 				for n in range(self.cardinality):
-					SETBIT(tmp.data.as_ulongs,
-							self.buf.data.as_ushorts[n])
+					SETBIT(tmp.data.as_ulongs, self.buf.data.as_ushorts[n])
+			self.buf = tmp
 			self.cardinality = bitsetintersectinplace(
-					tmp.data.as_ulongs,
-					other.buf.data.as_ulongs)
+					self.buf.data.as_ulongs, other.buf.data.as_ulongs)
 			self.state = DENSE
-			self.buf = tmp
 		elif self.state == POSITIVE and other.state == POSITIVE:
 			length = intersect2by2(
 					self.buf.data.as_ushorts,
@@ -148,9 +165,9 @@ cdef class Block(object):
 					BLOCKSIZE - self.cardinality,
 					BLOCKSIZE - other.cardinality,
 					tmp.data.as_ushorts)
-			array.resize(tmp, length)
-			self.cardinality = BLOCKSIZE - length
 			self.buf = tmp
+			array.resize(self.buf, length)
+			self.cardinality = BLOCKSIZE - length
 		elif self.state == INVERTED and other.state == POSITIVE:
 			tmp = array.clone(ushortarray,
 					max(BLOCKSIZE - self.cardinality, other.cardinality),
@@ -161,10 +178,10 @@ cdef class Block(object):
 					other.cardinality,
 					BLOCKSIZE - self.cardinality,
 					tmp.data.as_ushorts)
-			array.resize(tmp, length)
+			self.buf = tmp
+			array.resize(self.buf, length)
 			self.state = POSITIVE
 			self.cardinality = length
-			self.buf = tmp
 		elif self.state == POSITIVE and other.state == INVERTED:
 			length = difference(
 					self.buf.data.as_ushorts,
@@ -187,20 +204,22 @@ cdef class Block(object):
 						other.buf.data.as_ushorts,
 						self.cardinality, other.cardinality,
 						tmp.data.as_ushorts)
-				array.resize(tmp, self.cardinality)
 				self.buf = tmp
+				array.resize(self.buf, self.cardinality)
 			elif other.state == DENSE:
-				tmp = array.copy(other.buf)
+				tmp = array.clone(ushortarray, BITMAPSIZE // sizeof(short),
+						False)
+				memcpy(tmp.data.as_chars, other.buf.data.as_chars,
+						BITMAPSIZE)
 				length = other.cardinality
-				array.resize(tmp, length + self.cardinality)
 				for n in range(self.cardinality):
 					if not TESTBIT(tmp.data.as_ulongs,
 							self.buf.data.as_ushorts[n]):
+						SETBIT(tmp.data.as_ulongs, self.buf.data.as_ushorts[n])
 						length += 1
-					SETBIT(tmp.data.as_ulongs, self.buf.data.as_ushorts[n])
+				self.buf = tmp
 				self.state = DENSE
 				self.cardinality = length
-				self.buf = tmp
 			elif other.state == INVERTED:
 				tmp = array.clone(ushortarray, BLOCKSIZE - self.cardinality,
 						False)
@@ -210,10 +229,10 @@ cdef class Block(object):
 						BLOCKSIZE - other.cardinality,
 						self.cardinality,
 						tmp.data.as_ushorts)
-				array.resize(tmp, length)
+				self.buf = tmp
+				array.resize(self.buf, length)
 				self.state = INVERTED
 				self.cardinality = BLOCKSIZE - length
-				self.buf = tmp
 		elif self.state == DENSE:
 			if other.state == POSITIVE:
 				for n in range(other.cardinality):
@@ -235,10 +254,10 @@ cdef class Block(object):
 						tmp.data.as_ushorts[length] = (
 								other.buf.data.as_ushorts[n])
 						length += 1
-				array.resize(tmp, length)
+				self.buf = tmp
+				array.resize(self.buf, length)
 				self.cardinality = BLOCKSIZE - length
 				self.state = INVERTED
-				self.buf = tmp
 		elif self.state == INVERTED:
 			if other.state == POSITIVE:
 				length = difference(
@@ -258,9 +277,9 @@ cdef class Block(object):
 						tmp.data.as_ushorts[length] = (
 							self.buf.data.as_ushorts[n])
 						length += 1
-				array.resize(tmp, length)
-				self.cardinality = BLOCKSIZE - length
 				self.buf = tmp
+				array.resize(self.buf, length)
+				self.cardinality = BLOCKSIZE - length
 			elif other.state == INVERTED:
 				length = intersect2by2(self.buf.data.as_ushorts,
 						other.buf.data.as_ushorts,
@@ -322,9 +341,9 @@ cdef class Block(object):
 					BLOCKSIZE - self.cardinality,
 					BLOCKSIZE - other.cardinality,
 					tmp.data.as_ushorts)
-			array.resize(tmp, length)
-			self.cardinality = BLOCKSIZE - length
 			self.buf = tmp
+			array.resize(self.buf, length)
+			self.cardinality = BLOCKSIZE - length
 		elif self.state == INVERTED and other.state == POSITIVE:
 			length = intersect2by2(
 					self.buf.data.as_ushorts,
@@ -361,7 +380,7 @@ cdef class Block(object):
 				if TESTBIT(self.buf.data.as_ulongs,
 						other.buf.data.as_ushorts[n]):
 					CLEARBIT(self.buf.data.as_ulongs,
-						self.buf.data.as_ushorts[n])
+						other.buf.data.as_ushorts[n])
 					self.cardinality -= 1
 		elif self.state == INVERTED and other.state == DENSE:
 			self.todense()
@@ -381,9 +400,9 @@ cdef class Block(object):
 					self.cardinality,
 					other.cardinality,
 					tmp.data.as_ushorts)
-			array.resize(tmp, length)
-			self.cardinality = length
 			self.buf = tmp
+			array.resize(self.buf, length)
+			self.cardinality = length
 		elif self.state == INVERTED and other.state == INVERTED:
 			tmp = array.clone(ushortarray,
 					BLOCKSIZE - (self.cardinality + other.cardinality),
@@ -394,9 +413,9 @@ cdef class Block(object):
 					BLOCKSIZE - self.cardinality,
 					BLOCKSIZE - other.cardinality,
 					tmp.data.as_ushorts)
-			array.resize(tmp, length)
-			self.cardinality = BLOCKSIZE - length
 			self.buf = tmp
+			array.resize(self.buf, length)
+			self.cardinality = BLOCKSIZE - length
 		elif self.state == INVERTED and other.state == POSITIVE:
 			self.todense()
 			self.ixor(other)
@@ -407,44 +426,6 @@ cdef class Block(object):
 			return
 		self.resize()
 
-	cdef flip(self):
-		"""In-place complement of this block."""
-		if self.state == DENSE:
-			for n in range(BITNSLOTS(BLOCKSIZE)):
-				self.buf.data.as_ulongs[n] = ~self.buf.data.as_ulongs[n]
-		elif self.state == POSITIVE:
-			self.state = INVERTED
-		elif self.state == INVERTED:
-			self.state = POSITIVE
-		# FIXME: need notion of maximium element
-		self.cardinality = BLOCKSIZE - self.cardinality
-
-	cdef resize(self):
-		"""Convert between dense, sparse, inverted sparse as needed."""
-		if self.state == DENSE:
-			if self.cardinality < MAXARRAYLENGTH:
-				self.toposarray()
-			elif self.cardinality > BLOCKSIZE - MAXARRAYLENGTH:
-				self.toinvarray()
-		elif self.state == INVERTED:
-			if (MAXARRAYLENGTH < self.cardinality
-					< BLOCKSIZE - MAXARRAYLENGTH):
-				self.todense()
-			elif self.cardinality < MAXARRAYLENGTH:
-				# shouldn't happen?
-				raise ValueError
-		elif self.state == POSITIVE:
-			if MAXARRAYLENGTH < self.cardinality < BLOCKSIZE - MAXARRAYLENGTH:
-				# To dense bitvector
-				self.todense()
-			elif self.cardinality > BLOCKSIZE - MAXARRAYLENGTH:
-				# shouldn't happen?
-				raise ValueError
-
-	def size(self):
-		"""Return memory usage."""
-		return 2 * len(self.buf)
-
 	cdef issubset(self, Block other):
 		cdef int n, m = 0
 		if self.key != other.key or self.cardinality > other.cardinality:
@@ -594,7 +575,7 @@ cdef class Block(object):
 		elif self.state == POSITIVE:
 			return self.buf.data.as_ushorts[i]
 		elif self.state == DENSE:
-			for n in range(BITMAPSIZE):
+			for n in range(BITNSLOTS(BLOCKSIZE)):
 				w = bit_popcount(self.buf.data.as_ulongs[n])
 				if w > i:
 					return BITSIZE * n + select64(
@@ -608,18 +589,55 @@ cdef class Block(object):
 			return self.buf.data.as_ushorts[len(self.buf) - 1] + (
 					i - len(self.buf))
 
+	cdef flip(self):
+		"""In-place complement of this block."""
+		if self.state == DENSE:
+			for n in range(BITNSLOTS(BLOCKSIZE)):
+				self.buf.data.as_ulongs[n] = ~self.buf.data.as_ulongs[n]
+		elif self.state == POSITIVE:
+			self.state = INVERTED
+		elif self.state == INVERTED:
+			self.state = POSITIVE
+		# FIXME: need notion of maximium element
+		self.cardinality = BLOCKSIZE - self.cardinality
+
+	cdef resize(self):
+		"""Convert between dense, sparse, inverted sparse as needed."""
+		if self.state == DENSE:
+			if self.cardinality < MAXARRAYLENGTH:
+				self.toposarray()
+			elif self.cardinality > BLOCKSIZE - MAXARRAYLENGTH:
+				self.toinvarray()
+		elif self.state == INVERTED:
+			if (MAXARRAYLENGTH < self.cardinality
+					< BLOCKSIZE - MAXARRAYLENGTH):
+				self.todense()
+			elif self.cardinality < MAXARRAYLENGTH:
+				# shouldn't happen?
+				raise ValueError
+		elif self.state == POSITIVE:
+			if MAXARRAYLENGTH < self.cardinality < BLOCKSIZE - MAXARRAYLENGTH:
+				# To dense bitvector
+				self.todense()
+			elif self.cardinality > BLOCKSIZE - MAXARRAYLENGTH:
+				# shouldn't happen?
+				raise ValueError
+
+	def __sizeof__(self):
+		"""Return memory usage in bytes."""
+		return (sizeof(self.state) + sizeof(self.key)
+				+ sizeof(self.cardinality) + sys.getsizeof(self.buf))
+
 	cdef todense(self):
 		# To dense bitvector
-		cdef array.array tmp
 		cdef int n
+		tmp = array.clone(ushortarray, BITMAPSIZE // sizeof(short), False)
 		if self.state == INVERTED:
-			tmp = array.clone(ushortarray, BITMAPSIZE, False)
-			memset(tmp.data.as_uchars, 255, BLOCKSIZE // sizeof(char))
+			memset(tmp.data.as_chars, 255, BITMAPSIZE)
 			for n in range(BLOCKSIZE - self.cardinality):
-				CLEARBIT(tmp.data.as_ulongs,
-						self.buf.data.as_ushorts[n])
+				CLEARBIT(tmp.data.as_ulongs, self.buf.data.as_ushorts[n])
 		else:  # self.state == POSITIVE:
-			tmp = array.clone(ushortarray, BITMAPSIZE, True)
+			memset(tmp.data.as_chars, 0, BITMAPSIZE)
 			for n in range(self.cardinality):
 				SETBIT(tmp.data.as_ulongs, self.buf.data.as_ushorts[n])
 		self.state = DENSE
@@ -755,15 +773,21 @@ cdef class Block(object):
 		elif self.state == POSITIVE:
 			return 'array(%r)' % self.buf
 
+	cdef allocarray(self):
+		self.buf = array.clone(ushortarray, 0, False)
+
 	def __reduce__(self):
-		return (Block, None, (self.key, self.state,
-				self.cardinality, self.buf))
-
-	def __setstate__(self, key, state, cardinality, buf):
-		self.key = key
-		self.state = state
-		self.cardinality = cardinality
-		self.buf = buf
+		return (Block, (), dict(
+				key=self.key,
+				state=self.state,
+				cardinality=self.cardinality,
+				buf=self.buf))
+
+	def __setstate__(self, state):
+		self.key = state['key']
+		self.state = state['state']
+		self.cardinality = state['cardinality']
+		self.buf = state['buf']
 
 
 cdef inline int min(int a, int b):

{roaringbitmap-0.1 → roaringbitmap-0.3}/src/macros.h

@@ -1,7 +1,7 @@
 
 /* http://c-faq.com/misc/bitsets.html */
 #define BITSIZE 			(8 * sizeof(uint64_t))
-#define BITMASK(b) 			(1UL << ((b) % BITSIZE))
+#define BITMASK(b) 			(1ULL << ((b) % BITSIZE))
 #define BITSLOT(b) 			((b) / BITSIZE)
 #define SETBIT(a, b) 		((a)[BITSLOT(b)] |= BITMASK(b))
 #define TOGGLEBIT(a, b)		((a)[BITSLOT(b)] ^= BITMASK(b))