yencode 1.1.3 → 1.1.4

package/src/platform.cc

@@ -2,12 +2,6 @@
 #ifdef PLATFORM_ARM
 # ifdef __ANDROID__
 #  include <cpu-features.h>
-# elif defined(__linux__) || (defined(__FreeBSD__) && __FreeBSD__ >= 12)
-#  include <sys/auxv.h>
-#  include <asm/hwcap.h>
-# elif (defined(__FreeBSD__) && __FreeBSD__ < 12)
-#  include <sys/sysctl.h>
-#  include <asm/hwcap.h>
 # elif defined(_WIN32)
 #  define WIN32_LEAN_AND_MEAN
 #  define NOMINMAX
@@ -15,6 +9,13 @@
 # elif defined(__APPLE__)
 #  include <sys/types.h>
 #  include <sys/sysctl.h>
+# elif defined(__has_include)
+#  if __has_include(<sys/auxv.h>)
+#   include <sys/auxv.h>
+#   if __has_include(<asm/hwcap.h>)
+#    include <asm/hwcap.h>
+#   endif
+#  endif
 # endif
 bool cpu_supports_neon() {
 # if defined(AT_HWCAP)
@@ -48,7 +49,11 @@ bool cpu_supports_neon() {
 		return false;
 	return (bool)supported;
 # endif
-	return true; // assume NEON support, if compiled as such, otherwise (I think Windows and iOS require it)
+# ifdef __aarch64__
+	return true; // assume NEON support on AArch64
+# else
+	return false;
+# endif
 }
 #endif
 
@@ -121,13 +126,32 @@ int cpu_supports_isa() {
 			return ret | ISA_LEVEL_SSSE3;
 		
 		if(flags[2] & 0x80000) { // SSE4.1
-			if((flags[2] & 0x18800000) == 0x18800000) { // POPCNT + OSXSAVE + AVX
+			if((flags[2] & 0x1C800000) == 0x1C800000) { // POPCNT + OSXSAVE + XSAVE + AVX
 				int xcr = _GET_XCR() & 0xff; // ignore unused bits
 				if((xcr & 6) == 6) { // AVX enabled
 					int cpuInfo[4];
 					_cpuidX(cpuInfo, 7, 0);
 					if((cpuInfo[1] & 0x128) == 0x128 && (ret & ISA_FEATURE_LZCNT)) { // BMI2 + AVX2 + BMI1
+						
+						// check AVX10
+						int cpuInfo2[4];
+						_cpuidX(cpuInfo2, 7, 1);
+						if(cpuInfo2[3] & 0x80000) {
+							_cpuidX(cpuInfo2, 0x24, 0);
+							if((cpuInfo2[2] & 0xff) >= 1 && ( // minimum AVX10.1
+#ifdef YENC_DISABLE_AVX256
+								cpuInfo2[2] & 0x10000 // AVX10/128
+#else
+								cpuInfo2[2] & 0x20000 // AVX10/256
+#endif
+							)) {
+								if(((xcr & 0xE0) == 0xE0) && (cpuInfo2[2] & 0x40000)) ret |= ISA_FEATURE_EVEX512;
+								return ret | ISA_LEVEL_VBMI2;
+							}
+						}
+						
 						if(((xcr & 0xE0) == 0xE0) && (cpuInfo[1] & 0xC0010000) == 0xC0010000) { // AVX512BW + AVX512VL + AVX512F
+							ret |= ISA_FEATURE_EVEX512;
 							if(cpuInfo[2] & 0x40)
 								return ret | ISA_LEVEL_VBMI2;
 							return ret | ISA_LEVEL_AVX3;
@@ -150,7 +174,7 @@ int cpu_supports_crc_isa() {
 	_cpuid1(flags);
 	
 	if((flags[2] & 0x80202) == 0x80202) { // SSE4.1 + SSSE3 + CLMUL
-		if((flags[2] & 0x18000000) == 0x18000000) { // OSXSAVE + AVX
+		if((flags[2] & 0x1C000000) == 0x1C000000) { // AVX + OSXSAVE + XSAVE
 			int xcr = _GET_XCR() & 0xff; // ignore unused bits
 			if((xcr & 6) == 6) { // AVX enabled
 				int cpuInfo[4];
@@ -166,3 +190,27 @@ int cpu_supports_crc_isa() {
 }
 
 #endif // PLATFORM_X86
+
+#ifdef __riscv
+# if defined(__has_include)
+#  if __has_include(<sys/auxv.h>)
+#   include <sys/auxv.h>
+#   if __has_include(<asm/hwcap.h>)
+#    include <asm/hwcap.h>
+#   endif
+#  endif
+# endif
+bool cpu_supports_rvv() {
+# if defined(AT_HWCAP)
+	unsigned long ret;
+#  ifdef __FreeBSD__
+	elf_aux_info(AT_HWCAP, &ret, sizeof(ret));
+#  else
+	ret = getauxval(AT_HWCAP);
+#  endif
+	return (ret & 0x20112D) == 0x20112D; // IMAFDCV; TODO: how to detect Z* features of 'G'?
+# endif
+	return false;
+}
+#endif
+

package/src/test_alignalloc.c

@@ -0,0 +1,6 @@
+// used by GYP to test if posix_memalign needs declarations
+#include "common.h"
+void f() {
+	void* p;
+	ALIGN_ALLOC(p, 16, 16);
+}

package/test/_speedbase.js

@@ -5,31 +5,32 @@ var trials = 8;
 var asyncWait = 1000;
 
 var maxSize = require('./_maxsize');
-var decimal = (''+1.1).substr(1, 1);
+var allocBuffer = (Buffer.allocUnsafe || Buffer);
+var decimal = (''+1.1).substring(1, 2);
 var fmtSpeed = function(size, time) {
 	var rate = ('' + (Math.round(100*(size/1048576)/time)/100)).split(decimal);
 	
-	return ('        ' + rate[0]).substr(-8) + decimal + ((rate[1]|0) + '00').substr(0, 2) + ' MiB/s';
+	return ('        ' + rate[0]).slice(-8) + decimal + ((rate[1]|0) + '00').substring(0, 2) + ' MiB/s';
 };
 var initBuffers = function() {
-	module.exports.bufWorst = new Buffer(sz);
-	module.exports.bufBest = new Buffer(sz);
+	module.exports.bufWorst = allocBuffer(sz);
+	module.exports.bufBest = allocBuffer(sz);
 	module.exports.bufAvg = [];
 	module.exports.bufAvg2x = [];
-	module.exports.bufTarget = new Buffer(maxSize(sz));
+	module.exports.bufTarget = allocBuffer(maxSize(sz));
 
 	module.exports.bufWorst.fill(224);
 	module.exports.bufBest.fill(0);
 	
 	// use cipher as a fast, consistent RNG
-	var cipher = require('crypto').createCipher;
-	[['aes-128-cbc', 'my_incredibly_strong_password'],
-	 ['rc4', 'my_incredibly_strong_password'],
-	 ['aes-128-cbc', '9h8a08b08qpklnac']
+	var cipher = require('crypto').createCipheriv;
+	[['aes-128-cbc', 'my_incredible_pw', '                '],
+	 ['aes-128-cbc', 'nfa807g2lablzxk1', '                '],
+	 ['aes-128-cbc', '3h89sdg923jnkbas', '                ']
 	].forEach(function(cargs) {
 		var rand = cipher.apply(null, cargs);
 		var data = Buffer.concat([rand.update(module.exports.bufBest), rand.final()]).slice(0, sz);
-		module.exports.bufAvg.push(new Buffer(data));
+		module.exports.bufAvg.push(Buffer.concat([data]));
 		
 		// all yEnc special characters exist in range 0-61 (post shift) or 214-19 (pre-shift)
 		// to generate biased data, we'll pack the range down (64-191 will get packed to 192-63)
@@ -83,7 +84,7 @@ module.exports = {
 	run: function(name, fn, sz2) {
 		var time = module.exports.bench(fn);
 		console.log(
-			(name+'                         ').substr(0, 25) + ':'
+			(name+'                         ').substring(0, 25) + ':'
 			+ fmtSpeed(sz*rounds, time)
 			+ (sz2 ? (' ' + fmtSpeed(sz2*rounds, time)) : '')
 		);

package/test/speeddec.js

@@ -1,17 +1,18 @@
 var y = require('../build/Release/yencode');
 var _ = require('./_speedbase');
+var allocBuffer = (Buffer.allocUnsafe || Buffer);
 
 var bufSize = _.bufTarget.length;
 
-var mWorst = new Buffer(bufSize);
+var mWorst = allocBuffer(bufSize);
 var mAvg = _.bufAvg.map(function() {
-	return new Buffer(bufSize);
+	return allocBuffer(bufSize);
 });
 var mAvg2x = _.bufAvg2x.map(function() {
-	return new Buffer(bufSize);
+	return allocBuffer(bufSize);
 });
-var mBest = new Buffer(bufSize);
-var mBest2 = new Buffer(_.size);
+var mBest = allocBuffer(bufSize);
+var mBest2 = allocBuffer(_.size);
 mBest2.fill(32);
 
 var lenWorst = y.encodeTo(_.bufWorst, mWorst);

package/test/testdec.js

@@ -1,5 +1,8 @@
 // a basic script to test that raw yEnc works as expected
 
+var allocBuffer = (Buffer.allocUnsafe || Buffer);
+var toBuffer = (Buffer.alloc ? Buffer.from : Buffer);
+
 var assert = require('assert');
 var y = (function() {
 	try {
@@ -14,12 +17,12 @@ var ord = function(c) {
 // slow reference yEnc implementation
 var refYDec = function(src, findEnd) {
 	var ret = [];
-	if(findEnd && chr(src[0]) == '=' && chr(src[1]) == 'y') return new Buffer(0);
+	if(findEnd && chr(src[0]) == '=' && chr(src[1]) == 'y') return allocBuffer(0);
 	for (var i = 0; i < src.length; i++) {
 		switch(chr(src[i])) {
 			case '\r':
 				if(findEnd && chr(src[i+1]) == '\n' && chr(src[i+2]) == '=' && chr(src[i+3]) == 'y')
-					return new Buffer(ret);
+					return toBuffer(ret);
 			case '\n': continue;
 			case '=':
 				i++;
@@ -30,7 +33,7 @@ var refYDec = function(src, findEnd) {
 		}
 		ret.push((src[i] - 42) & 0xFF);
 	}
-	return new Buffer(ret);
+	return toBuffer(ret);
 };
 var refYDecRaw = function(src, findEnd) {
 	// undo NNTP layer
@@ -39,7 +42,7 @@ var refYDecRaw = function(src, findEnd) {
 	if(src[0] == ord('.')) {
 		i++;
 		if(findEnd && src[1] == ord('\r') && src[2] == ord('\n'))
-			return new Buffer(0);
+			return allocBuffer(0);
 	}
 	// TODO: do leading/trailing spaces/tabs need to be trimmed?
 	for(; i<src.length; i++) {
@@ -61,17 +64,24 @@ var refYDecRaw = function(src, findEnd) {
 var testFuncs = [
 	{l: 'nntp', r: refYDecRaw, a: function(s) {
 		return y.decode(s, true);
+	}, i: function(b) {
+		return y.decodeTo(b, b, true);
+	}},
+	{l: 'plain', r: refYDec, a: y.decode, i: function(b) {
+		return y.decodeTo(b, b, false);
 	}},
-	{l: 'plain', r: refYDec, a: y.decode},
 	{l: 'nntp-end', r: function(s) {
 		return refYDecRaw(s, true);
 	}, a: function(s) {
 		if(!s.length) return Buffer(0);
 		return y.decodeIncr(s).output;
+	}, i: function(b) {
+		if(!b.length) return 0;
+		return y.decodeIncr(b, 0, b).written;
 	}}
 ];
 var doTest = function(msg, data, expected) {
-	data = new Buffer(data);
+	data = toBuffer(data);
 	
 	var prepad = 48, postpad = 48;
 	if(data.length > 1024) {
@@ -80,17 +90,17 @@ var doTest = function(msg, data, expected) {
 	}
 	
 	for(var i=0; i<prepad; i++) {
-		var pre = new Buffer(i);
+		var pre = allocBuffer(i);
 		pre.fill(1);
 		for(var j=0; j<postpad; j++) {
 			testFuncs.forEach(function(f) {
-				var post = new Buffer(j);
+				var post = allocBuffer(j);
 				post.fill(1);
 				
 				var testData = Buffer.concat([pre, data, post]);
 				var x;
 				if(expected === undefined) x = f.r(testData).toString('hex');
-				else x = new Buffer(expected).toString('hex').replace(/ /g, '');
+				else x = toBuffer(expected).toString('hex').replace(/ /g, '');
 				var actual = f.a(testData).toString('hex');
 				if(actual != x) {
 					console.log('Actual:', actual);
@@ -98,6 +108,14 @@ var doTest = function(msg, data, expected) {
 					console.log('Source:', data.toString('hex'));
 					assert.equal(actual, x, msg + ' [' + i + '/' + j + ' ' + f.l + ']');
 				}
+				// also test in-situ decoding
+				var ilen = f.i(testData);
+				actual = testData.slice(0, ilen).toString('hex');
+				if(actual != x) {
+					console.log('Actual:', actual);
+					console.log('Expect:', x);
+					assert.equal(actual, x, msg + ' (in-situ) [' + i + '/' + j + ' ' + f.l + ']');
+				}
 			});
 			if(expected !== undefined) return; // if given expected string, only do one test
 			
@@ -137,7 +155,7 @@ doTest('Bad escape, Yenc end sequence', '=\r\n=y');
 
 
 // longer tests
-var b = new Buffer(256);
+var b = allocBuffer(256);
 b.fill(0);
 doTest('Long no escaping', b);
 b.fill('='.charCodeAt(0));
@@ -155,9 +173,7 @@ b.fill(223);
 doTest('Long all tabs', b);
 
 // test for past bug in ARMv8 NEON decoder where nextMask wasn't properly compensated for
-doTest('Extra null issue', new Buffer('2e900a4fb6054c9126171cdc196dc41237bb1b76da9191aa5e85c1d2a2a5c638fe39054a210e8c799473cd510541fd118f3904b242a9938558c879238aae1d3bdab32e287cedb820b494f54ffae6dd0b13f73a4a9499df486a7845c612182bcef72a6e50a8e98351c35765d26c605115dc8c5c56a5e3f20ae6da8dcd78536e6d1601eb1fc3ddc774', 'hex'));
-// end detection bug
-doTest('End detect', new Buffer('612e6161610d610d612e793d3d0d0d2e612e2e0a0d0d61792e3d3d61612e0d0a2e0d2e0a0d79612e0a3d2e2e793d2e610a0d0a0a2e793d790d612e61612e0a3d792e2e3d2e7961793d792e0a61790a0d0a2e0d0a3d0a0d0d0d0a610a0a6161792e3d2e0a2e0d0d0d613d610a0a0a793d613d3d0a3d790d3d0a0a2e2e7979796179613d0d2e792e793d3d61792e612e2e2e793d616161790d0d2e0d0d793d0d790a0a3d0d617979790d2e0d792e612e610a0a0a0a0a79790d0a610d612e0d0a0d3d0a61792e2e0a790d0d792e790d0a2e79612e3d0a79790a0d0d3d0a0a0d3d0a7961610a2e613d792e0a612e613d610a2e0a0a79613d2e2e0d3d3d2e793d792e792e0d0d610d2e2e0d2e79610d2e790d790d3d2e3d790a0a0d0a0a0a612e2e79612e0d2e3d793d2e0a2e3d790a2e3d792e2e610d3d2e0d3d3d0a3d2e0d613d2e0d61610a3d0a2e0a0a3d3d612e3d790d6161613d3d612e3d0d0a2e0d0d0d616179790a2e3d610d612e0d2e3d0d0a3d610d0d61610a7961613d2e790d613d610a3d612e0a2e0d79790d0a610a2e2e0a2e612e2e0d792e61610a2e0d610d3d0a793d613d0d3d0a3d0d0a613d2e0a3d610a3d0d793d0d7979792e3d613d0a2e61610d793d2e0a0a2e612e0d2e2e792e0d2e613d0d790a0d2e610d2e0a2e61793d0d0d0a0a0d0d2e2e0d2e793d3d79612e0a610a610a0d610d0d2e2e790', 'hex'));
+doTest('Extra null issue', toBuffer('2e900a4fb6054c9126171cdc196dc41237bb1b76da9191aa5e85c1d2a2a5c638fe39054a210e8c799473cd510541fd118f3904b242a9938558c879238aae1d3bdab32e287cedb820b494f54ffae6dd0b13f73a4a9499df486a7845c612182bcef72a6e50a8e98351c35765d26c605115dc8c5c56a5e3f20ae6da8dcd78536e6d1601eb1fc3ddc774', 'hex'));
 
 // random tests
 for(var i=0; i<32; i++) {
@@ -168,7 +184,7 @@ for(var i=0; i<32; i++) {
 // targeted random tests
 var charset = '=\r\n.ay';
 var randStr = function(n) {
-	var ret = new Buffer(n);
+	var ret = allocBuffer(n);
 	for(var i=0; i<n; i++)
 		ret[i] = ord(charset[(Math.random() * charset.length) | 0]);
 	return ret;

package/test/testenc.js

@@ -1,6 +1,9 @@
 // a basic script to test that raw yEnc works as expected
 
 var assert = require('assert');
+var allocBuffer = (Buffer.allocUnsafe || Buffer);
+var toBuffer = (Buffer.alloc ? Buffer.from : Buffer);
+
 var y = (function() {
 	try {
 		return require('../build/Debug/yencode.node');
@@ -47,10 +50,10 @@ var refYEnc = function(src, line_size, col) {
 		ret.push((c+64) & 0xFF);
 	}
 	
-	return new Buffer(ret);
+	return toBuffer(ret);
 };
 var doTest = function(msg, test, expected) {
-	test[0] = new Buffer(test[0]);
+	test[0] = toBuffer(test[0]);
 	if(!test[1]) test[1] = 128; // line size
 	if(!test[2]) test[2] = 0; // column offset
 	
@@ -64,7 +67,7 @@ var doTest = function(msg, test, expected) {
 		assert.equal(actual, expected, msg);
 	}
 	
-	var buf = new Buffer(maxSize(test[0].length, test[1]));
+	var buf = allocBuffer(maxSize(test[0].length, test[1]));
 	var len = y.encodeTo.apply(null, [test[0], buf].concat(test.slice(1)));
 	assert.equal(buf.slice(0, len).toString('hex'), expected, msg);
 };
@@ -97,7 +100,7 @@ var runLineSizes = function(fn) {
 // simple tests
 runLineSizes(function(ls, offs) {
 	var infoStr = ' [ls='+ls+', offs='+offs+']';
-	var b = new Buffer(256);
+	var b = allocBuffer(256);
 	b.fill(0);
 	doTest('Long no escaping' + infoStr, [b, ls]);
 	b.fill(227);
@@ -110,7 +113,7 @@ runLineSizes(function(ls, offs) {
 
 // case tests
 var padLens = range(0, 35).concat(range(46, 51)).concat(range(62, 67));
-var padding = new Buffer(128);
+var padding = allocBuffer(128);
 padding.fill(97); // 'a'
 [ ['Empty test', [[]], ''],
   ['Simple test', [[0,1,2,3,224,4]]],
@@ -127,7 +130,7 @@ padding.fill(97); // 'a'
 		// if no defined result, try padding the data to make it long enough to invoke SIMD behaviour
 		padLens.forEach(function(prepad) {
 			padLens.forEach(function(postpad) {
-				var newBuf = Buffer.concat([padding.slice(0, prepad), new Buffer(test[1][0]), padding.slice(0, postpad)]);
+				var newBuf = Buffer.concat([padding.slice(0, prepad), toBuffer(test[1][0]), padding.slice(0, postpad)]);
 				var newTest = test.slice();
 				newTest[1] = [newBuf].concat(newTest[1].slice(1));
 				doTest.apply(null, newTest);
@@ -147,7 +150,7 @@ for(var i=0; i<32; i++) {
 // targeted random tests
 var charset = 'a\xd6\xdf\xe0\xe3\xf6\x04\x13';
 var randStr = function(n) {
-	var ret = new Buffer(n);
+	var ret = allocBuffer(n);
 	for(var i=0; i<n; i++)
 		ret[i] = charset[(Math.random() * charset.length) | 0].charCodeAt(0);
 	return ret;

package/test/testpostdec.js

@@ -1,6 +1,7 @@
 
 var assert = require('assert');
 var y = require('../index.js');
+var toBuffer = (Buffer.alloc ? Buffer.from : Buffer);
 
 var assertObjectHas = function(obj, required) {
 	for(var k in required) {
@@ -23,7 +24,7 @@ postData = [
 	'=yend size=10 part=5 pcrc32=97f4bd52',
 	''
 ].join('\r\n');
-post = y.from_post(new Buffer(postData));
+post = y.from_post(toBuffer(postData));
 assertObjectHas(post, {
 	yencStart: 0,
 	dataStart: postData.indexOf('...'),
@@ -59,7 +60,7 @@ postData = [
 	'.... data',
 	'=yend size=2 pcrc32=invalid pcrc32=invalid invalid_prop',
 ].join('\r\n');
-post = y.from_post(new Buffer(postData));
+post = y.from_post(toBuffer(postData));
 assertObjectHas(post, {
 	yencStart: postData.indexOf('=ybegin'),
 	dataStart: postData.indexOf('...'),
@@ -97,7 +98,7 @@ postData = [
 	'=yend size=0',
 	''
 ].join('\r\n');
-post = y.from_post(new Buffer(postData));
+post = y.from_post(toBuffer(postData));
 assertObjectHas(post, {
 	yencStart: 0,
 	dataStart: undefined,
@@ -117,9 +118,9 @@ assertObjectHas(post, {
 
 
 // test parse errors
-post = y.from_post(new Buffer('invalid post'));
+post = y.from_post(toBuffer('invalid post'));
 assert.equal(post.code, 'no_start_found');
-post = y.from_post(new Buffer('=ybegin abc=def'));
+post = y.from_post(toBuffer('=ybegin abc=def'));
 assert.equal(post.code, 'no_end_found');