yencode 1.1.4 → 1.2.0

package/src/encoder.h

@@ -10,8 +10,12 @@ extern "C" {
 #include "hedley.h"
 
 extern size_t (*_do_encode)(int, int*, const unsigned char* HEDLEY_RESTRICT, unsigned char* HEDLEY_RESTRICT, size_t, int);
+extern int _encode_isa;
 #define do_encode (*_do_encode)
 void encoder_init();
+static inline int encode_isa_level() {
+	return _encode_isa;
+}
 
 
 

package/src/encoder_avx.cc

@@ -6,6 +6,7 @@
 void encoder_avx_init() {
 	_do_encode = &do_encode_simd< do_encode_sse<ISA_LEVEL_SSE4_POPCNT> >;
 	encoder_sse_lut<ISA_LEVEL_SSE4_POPCNT>();
+	_encode_isa = ISA_LEVEL_AVX;
 }
 #else
 void encoder_ssse3_init();

package/src/encoder_avx2.cc

@@ -6,6 +6,7 @@
 void encoder_avx2_init() {
 	_do_encode = &do_encode_simd< do_encode_avx2<ISA_LEVEL_AVX2> >;
 	encoder_avx2_lut<ISA_LEVEL_AVX2>();
+	_encode_isa = ISA_LEVEL_AVX2;
 }
 #else
 void encoder_avx_init();

package/src/encoder_common.h

@@ -2,26 +2,8 @@
 #define __YENC_ENCODER_COMMON
 
 // lookup tables for scalar processing
-#define _B1(n) _B(n), _B(n+1), _B(n+2), _B(n+3)
-#define _B2(n) _B1(n), _B1(n+4), _B1(n+8), _B1(n+12)
-#define _B3(n) _B2(n), _B2(n+16), _B2(n+32), _B2(n+48)
-#define _BX _B3(0), _B3(64), _B3(128), _B3(192)
-
-static const unsigned char escapeLUT[256] = { // whether or not the character is critical
-#define _B(n) ((n == 214 || n == '\r'+214 || n == '\n'+214 || n == '='-42) ? 0 : (n+42) & 0xff)
-	_BX
-#undef _B
-};
-static const uint16_t escapedLUT[256] = { // escaped sequences for characters that need escaping
-#define _B(n) ((n == 214 || n == 214+'\r' || n == 214+'\n' || n == '='-42 || n == 214+'\t' || n == 214+' ' || n == '.'-42) ? UINT16_PACK('=', ((n+42+64)&0xff)) : 0)
-	_BX
-#undef _B
-};
-
-#undef _B1
-#undef _B2
-#undef _B3
-#undef _BX
+extern const unsigned char escapeLUT[256];
+extern const uint16_t escapedLUT[256];
 
 
 size_t do_encode_generic(int line_size, int* colOffset, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, int doEnd);

package/src/encoder_neon.cc

@@ -520,6 +520,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_neon(int line_size, int* colOffset, const ui
 
 void encoder_neon_init() {
 	_do_encode = &do_encode_simd<do_encode_neon>;
+	_encode_isa = ISA_LEVEL_NEON;
 	// generate shuf LUT
 	for(int i=0; i<256; i++) {
 		int k = i;

package/src/encoder_rvv.cc

@@ -4,13 +4,6 @@
 #include "encoder.h"
 #include "encoder_common.h"
 
-# include <riscv_vector.h>
-# if defined(__clang__) && __clang_major__ < 16
-#  define RV(f) f
-# else
-#  define RV(f) __riscv_##f
-# endif
-
 
 static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RESTRICT _src, long& inpos, uint8_t*& outp, long& col, long lineSizeOffset) {
 	// TODO: vectorize
@@ -112,7 +105,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uin
 			vl2
 		);
 		
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
+#ifdef __riscv_v_intrinsic
 		data = RV(vor_vx_u8m2_mu)(cmp, tmpData, tmpData, 64, vl2);
 #else
 		data = RV(vor_vx_u8m2_m)(cmp, tmpData, tmpData, 64, vl2);
@@ -122,11 +115,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uin
 		size_t count = RV(vcpop_m_b4)(cmp, vl2);
 		if(count > 1) {
 			// widen mask: 4b->8b
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
-			vuint8mf4_t vcmp = RV(vlmul_trunc_v_u8m1_u8mf4)(RV(vreinterpret_v_b4_u8m1)(cmp));
-#else
-			vuint8mf4_t vcmp = *(vuint8mf4_t*)(&cmp);
-#endif
+			vuint8mf4_t vcmp = RV_VEC_U8MF4_CAST(cmp);
 			// TODO: use vwsll instead if available
 			//    -  is clmul useful here?
 			vuint8mf2_t xcmp = RV(vreinterpret_v_u16mf2_u8mf2)(RV(vwmulu_vx_u16mf2)(vcmp, 16, vl2));
@@ -134,11 +123,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uin
 			
 			// expand mask by inserting '1' between each bit (0000abcd -> 1a1b1c1d)
 			vuint8m1_t xcmpTmp = RV(vrgather_vv_u8m1)(MASK_EXPAND, RV(vlmul_ext_v_u8mf2_u8m1)(xcmp), vl2);
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
-			vbool2_t cmpmask = RV(vreinterpret_b2)(xcmpTmp);
-#else
-			vbool2_t cmpmask = *(vbool2_t*)(&xcmpTmp);
-#endif
+			vbool2_t cmpmask = RV_MASK_CAST(2, 8, xcmpTmp);
 			
 			// expand data and insert =
 			// TODO: use vwsll instead if available
@@ -149,7 +134,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uin
 			// prune unneeded =
 			vuint8m4_t dataTmp = RV(vreinterpret_v_u16m4_u8m4)(data2);
 			vuint8m4_t final_data = RV(vcompress_vm_u8m4)(
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
+#ifdef __riscv_v_intrinsic
 				dataTmp, cmpmask, vl2*2
 #else
 				cmpmask, dataTmp, dataTmp, vl2*2
@@ -213,6 +198,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uin
 
 void encoder_rvv_init() {
 	_do_encode = &do_encode_simd<do_encode_rvv>;
+	_encode_isa = ISA_LEVEL_RVV;
 }
 #else
 void encoder_rvv_init() {}

package/src/encoder_sse2.cc

@@ -6,6 +6,7 @@
 void encoder_sse2_init() {
 	_do_encode = &do_encode_simd< do_encode_sse<ISA_LEVEL_SSE2> >;
 	encoder_sse_lut<ISA_LEVEL_SSE2>();
+	_encode_isa = ISA_LEVEL_SSE2;
 }
 #else
 void encoder_sse2_init() {}

package/src/encoder_ssse3.cc

@@ -8,6 +8,7 @@
 void encoder_ssse3_init() {
 	_do_encode = &do_encode_simd< do_encode_sse<ISA_LEVEL_SSSE3> >;
 	encoder_sse_lut<ISA_LEVEL_SSSE3>();
+	_encode_isa = ISA_LEVEL_SSSE3;
 }
 #else
 void encoder_sse2_init();

package/src/encoder_vbmi2.cc

@@ -14,12 +14,14 @@ const bool encoder_has_avx10 = false;
 void encoder_vbmi2_init() {
 	_do_encode = &do_encode_simd< do_encode_avx2<ISA_LEVEL_VBMI2> >;
 	encoder_avx2_lut<ISA_LEVEL_VBMI2>();
+	_encode_isa = ISA_LEVEL_VBMI2;
 }
 # else
 #  include "encoder_sse_base.h"
 void encoder_vbmi2_init() {
 	_do_encode = &do_encode_simd< do_encode_sse<ISA_LEVEL_VBMI2> >;
 	encoder_sse_lut<ISA_LEVEL_VBMI2>();
+	_encode_isa = ISA_LEVEL_VBMI2;
 }
 # endif
 #else

package/src/platform.cc

@@ -138,14 +138,14 @@ int cpu_supports_isa() {
 						_cpuidX(cpuInfo2, 7, 1);
 						if(cpuInfo2[3] & 0x80000) {
 							_cpuidX(cpuInfo2, 0x24, 0);
-							if((cpuInfo2[2] & 0xff) >= 1 && ( // minimum AVX10.1
+							if((cpuInfo2[1] & 0xff) >= 1 && ( // minimum AVX10.1
 #ifdef YENC_DISABLE_AVX256
-								cpuInfo2[2] & 0x10000 // AVX10/128
+								cpuInfo2[1] & 0x10000 // AVX10/128
 #else
-								cpuInfo2[2] & 0x20000 // AVX10/256
+								cpuInfo2[1] & 0x20000 // AVX10/256
 #endif
 							)) {
-								if(((xcr & 0xE0) == 0xE0) && (cpuInfo2[2] & 0x40000)) ret |= ISA_FEATURE_EVEX512;
+								if(((xcr & 0xE0) == 0xE0) && (cpuInfo2[1] & 0x40000)) ret |= ISA_FEATURE_EVEX512;
 								return ret | ISA_LEVEL_VBMI2;
 							}
 						}

package/src/yencode.cc

@@ -383,7 +383,7 @@ FUNC(CRC32Combine) {
 	uint32_t crc1 = read_crc32(args[0]), crc2 = read_crc32(args[1]);
 	size_t len = (size_t)ARG_TO_INT(args[2]);
 	
-	crc1 = do_crc32_combine(crc1, crc2, len);
+	crc1 = crc32_combine(crc1, crc2, len);
 	RETURN_VAL(pack_crc32(ISOLATE crc1));
 }
 
@@ -399,11 +399,51 @@ FUNC(CRC32Zeroes) {
 			RETURN_ERROR("Second argument must be a 4 byte buffer");
 		crc1 = read_crc32(args[1]);
 	}
-	size_t len = (size_t)ARG_TO_INT(args[0]);
-	crc1 = do_crc32_zeros(crc1, len);
+	int len = ARG_TO_INT(args[0]);
+	if(len < 0)
+		crc1 = crc32_unzero(crc1, -len);
+	else
+		crc1 = crc32_zeros(crc1, len);
 	RETURN_VAL(pack_crc32(ISOLATE crc1));
 }
 
+FUNC(CRC32Multiply) {
+	FUNC_START;
+	
+	if (args.Length() < 2)
+		RETURN_ERROR("At least 2 arguments required");
+	
+	if (!node::Buffer::HasInstance(args[0]) || node::Buffer::Length(args[0]) != 4
+	|| !node::Buffer::HasInstance(args[1]) || node::Buffer::Length(args[1]) != 4)
+		RETURN_ERROR("You must supply a 4 byte Buffer for the first two arguments");
+	
+	uint32_t crc1 = read_crc32(args[0]);
+	uint32_t crc2 = read_crc32(args[1]);
+	crc1 = crc32_multiply(crc1, crc2);
+	RETURN_VAL(pack_crc32(ISOLATE crc1));
+}
+
+FUNC(CRC32Shift) {
+	FUNC_START;
+	
+	if (args.Length() < 1)
+		RETURN_ERROR("At least 1 argument required");
+	
+	uint32_t crc1 = 0x80000000;
+	if (args.Length() >= 2) {
+		if (!node::Buffer::HasInstance(args[1]) || node::Buffer::Length(args[1]) != 4)
+			RETURN_ERROR("Second argument must be a 4 byte buffer");
+		crc1 = read_crc32(args[1]);
+	}
+	int n = ARG_TO_INT(args[0]);
+	if(n < 0)
+		crc1 = crc32_shift(crc1, ~crc32_powmod(-n));
+	else
+		crc1 = crc32_shift(crc1, crc32_powmod(n));
+	RETURN_VAL(pack_crc32(ISOLATE crc1));
+}
+
+
 static void init_all() {
 	encoder_init();
 	decoder_init();
@@ -436,6 +476,8 @@ void yencode_init(
 	NODE_SET_METHOD(exports, "crc32", CRC32);
 	NODE_SET_METHOD(exports, "crc32_combine", CRC32Combine);
 	NODE_SET_METHOD(exports, "crc32_zeroes", CRC32Zeroes);
+	NODE_SET_METHOD(exports, "crc32_multiply", CRC32Multiply);
+	NODE_SET_METHOD(exports, "crc32_shift", CRC32Shift);
 	
 #if NODE_VERSION_AT_LEAST(10, 7, 0)
 	uv_once(&init_once, init_all);

package/test/testcrc.js

@@ -14,7 +14,7 @@ var ycrc32 = function(s) {
 };
 var doTest = function(msg, f, test, expected) {
 	if(!Array.isArray(test)) test = [test];
-	test[0] = Buffer(test[0]);
+	if(!Buffer.isBuffer(test[0])) test[0] = Buffer(test[0]);
 	if(!expected && test.length == 1 && f == 'crc32') expected = crc32(test[0]).toString('hex');
 	else if(Buffer.isBuffer(expected)) expected = expected.toString('hex');
 	assert.equal(y[f].apply(null, test).toString('hex'), expected, msg);
@@ -45,6 +45,22 @@ assert.equal(y.crc32_zeroes(1, ycrc32('')).toString('hex'), 'd202ef8d', 'Zeroes-
 assert.equal(y.crc32_zeroes(0, ycrc32('z')).toString('hex'), crc32('z').toString('hex'), 'Zeroes Empty Join');
 assert.equal(y.crc32_zeroes(4, ycrc32('z')).toString('hex'), crc32('z\u0000\u0000\u0000\u0000').toString('hex'), 'Zeroes (4) Join');
 
+assert.equal(y.crc32_zeroes(4, y.crc32_zeroes(-4)).toString('hex'), '00000000', 'Zeroes (-4+4)');
+assert.equal(y.crc32_zeroes(-4, y.crc32_zeroes(4)).toString('hex'), '00000000', 'Zeroes (4-4)');
+assert.equal(y.crc32_zeroes(6, y.crc32_zeroes(-2, ycrc32('z'))).toString('hex'), crc32('z\u0000\u0000\u0000\u0000').toString('hex'), 'Zeroes (-2+6) Join');
+
+
+assert.equal(y.crc32_multiply(Buffer([1,2,3,4]), ycrc32('')).toString('hex'), '00000000', 'Multiply by 0');
+assert.equal(y.crc32_multiply(Buffer([0x80,0,0,0]), Buffer([0x80,0,0,0])).toString('hex'), '80000000', 'Multiply by 1');
+assert.equal(y.crc32_multiply(Buffer([1,2,3,4]), Buffer([5,6,7,8])).toString('hex'), '81e243a3', 'Multiply random');
+
+assert.equal(y.crc32_shift(0).toString('hex'), '80000000', '2^0');
+assert.equal(y.crc32_shift(1).toString('hex'), '40000000', '2^1');
+assert.equal(y.crc32_shift(2).toString('hex'), '20000000', '2^2');
+assert.equal(y.crc32_shift(-1).toString('hex'), 'db710641', '2^-1');
+assert.equal(y.crc32_shift(-1, y.crc32_shift(1)).toString('hex'), '80000000', '2^(1-1)');
+assert.equal(y.crc32_shift(4, y.crc32_shift(-2)).toString('hex'), '20000000', '2^(-2+4)');
+
 
 doTest('Random', 'crc32', 'fj[-oqijnw34-59n26 4345j8yn89032q78t9ab9gabh023quhoiBO Z GEB780a sdasdq2345673-98hq2-9348h-na9we8zdfgh-n9  8qwhn-098');
 doTest('Random Continue', 'crc32', ['KZSHZ5EDOVAmDdakZZOrGSUGGKSpCJoWH7M0MHy6ohnSzvHY4DjpxXmyfWYJQoJ7tKdNhGcuRVUzrgXM', ycrc32('BdenbmoBgiB10ZkeUBjrsZV3dg2Da2fhHqU9TMdi69AHhLRck3Nk60YuFBXh6lvtefBpjdTxbeEmsaEm')], crc32('BdenbmoBgiB10ZkeUBjrsZV3dg2Da2fhHqU9TMdi69AHhLRck3Nk60YuFBXh6lvtefBpjdTxbeEmsaEmKZSHZ5EDOVAmDdakZZOrGSUGGKSpCJoWH7M0MHy6ohnSzvHY4DjpxXmyfWYJQoJ7tKdNhGcuRVUzrgXM'));
@@ -52,11 +68,11 @@ doTest('Random Continue', 'crc32', ['KZSHZ5EDOVAmDdakZZOrGSUGGKSpCJoWH7M0MHy6ohn
 
 // random tests
 for(var i=1; i<128; i++) {
-	var rand = require('crypto').pseudoRandomBytes(i);
+	var rand = Buffer(require('crypto').randomBytes(i));  // Bun needs explicit Buffer for pseudoRandomBytes
 	doTest('Random Short Buffer', 'crc32', rand);
 }
 for(var i=0; i<32; i++) {
-	var rand = require('crypto').pseudoRandomBytes(100000);
+	var rand = Buffer(require('crypto').randomBytes(100000));
 	doTest('Random Buffer', 'crc32', rand);
 	
 	var split = Math.random()*rand.length;

package/test/testcrcfuncs.c

@@ -0,0 +1,53 @@
+// compile with: c++ -O -DYENC_DISABLE_CRCUTIL=1 testcrcfuncs.c ../src/crc.cc ../src/crc_*.cc ../src/platform.cc -o testcrcfuncs
+
+#include <stdint.h>
+#include <stddef.h>
+#include "../src/crc.h"
+#include <assert.h>
+
+int main(void) {
+	crc_init();
+	
+	// test crc32_bytepow
+	
+	assert(crc32_bytepow(0) == 0);
+	assert(crc32_bytepow(1) == 8);
+	assert(crc32_bytepow(2) == 16);
+	assert(crc32_bytepow((1<<29)-1) == 0xfffffff8);
+	assert(crc32_bytepow(1<<29) == 1);
+	assert(crc32_bytepow((1<<29)+1) == 9);
+	
+	uint32_t actual;
+	// just do an exhaustive search, since it's not that slow
+	for(uint64_t i=1<<29; i<((1ULL<<32) + 256); i++) {
+		uint32_t ref = (i*8) % 0xffffffff;
+		actual = crc32_bytepow(i);
+		if(ref == 0)
+			assert(actual == 0 || actual == 0xffffffff);
+		else
+			assert(ref == actual);
+	}
+	
+	assert(crc32_bytepow(1ULL<<60) == 0x80000000);
+	assert(crc32_bytepow(1ULL<<61) == 1);
+	assert(crc32_bytepow(1ULL<<62) == 2);
+	assert(crc32_bytepow(1ULL<<63) == 4);
+	actual = crc32_bytepow(~0ULL);
+	assert(actual == 0 || actual == 0xffffffff);
+	assert(crc32_bytepow(~0ULL -1) == 0xfffffff7);
+	assert(crc32_bytepow((1ULL<<63) -1) == 0xfffffffb);
+	assert(crc32_bytepow((1ULL<<63) +1) == 12);
+	
+	
+	// test crc32_mul2pow
+	assert(crc32_mul2pow(0, 0) == 0);
+	assert(crc32_mul2pow(0, 1) == 1);
+	assert(crc32_mul2pow(1, 0) == 0);
+	assert(crc32_mul2pow(1, 0x80000000) == 0x40000000);
+	assert(crc32_mul2pow(1, 0x40000000) == 0x20000000);
+	assert(crc32_mul2pow(4, 0x80000000) == 0x08000000);
+	assert(crc32_mul2pow(5, 0x80000000) == 0x04000000);
+	assert(crc32_mul2pow(0xffffffff, 123) == 123);
+	
+	return 0;
+}

package/test/testdec.js

@@ -114,6 +114,7 @@ var doTest = function(msg, data, expected) {
 				if(actual != x) {
 					console.log('Actual:', actual);
 					console.log('Expect:', x);
+					console.log('Source:', data.toString('hex'));
 					assert.equal(actual, x, msg + ' (in-situ) [' + i + '/' + j + ' ' + f.l + ']');
 				}
 			});
@@ -177,7 +178,7 @@ doTest('Extra null issue', toBuffer('2e900a4fb6054c9126171cdc196dc41237bb1b76da9
 
 // random tests
 for(var i=0; i<32; i++) {
-	var rand = require('crypto').pseudoRandomBytes(128*1024);
+	var rand = require('crypto').randomBytes(128*1024);
 	doTest('Random', rand);
 }
 

package/test/testenc.js

@@ -141,7 +141,7 @@ padding.fill(97); // 'a'
 
 // random tests
 for(var i=0; i<32; i++) {
-	var rand = require('crypto').pseudoRandomBytes(4*1024);
+	var rand = require('crypto').randomBytes(4*1024);
 	runLineSizes(function(ls, offs) {
 		doTest('Random [ls='+ls+', offs='+offs+']', [rand, ls, offs]);
 	});