yencode 1.2.1 → 1.2.4

package/binding.gyp

@@ -2,14 +2,15 @@
   "variables": {
     "enable_native_tuning%": 1,
     "disable_avx256%": 0,
-    "disable_crcutil%": 0
+    "disable_crcutil%": 0,
+    "built_with_electron%": 0
   },
   "target_defaults": {
     "conditions": [
       ['target_arch=="ia32"', {
         "msvs_settings": {"VCCLCompilerTool": {"EnableEnhancedInstructionSet": "2"}}
       }],
-      ['OS!="win" and enable_native_tuning!=0', {
+      ['OS!="win" and enable_native_tuning!=0 and target_arch==host_arch and (built_with_electron==0 or host_arch!="x64")', {
         "variables": {
           "supports_native%": "<!(<!(echo ${CXX_target:-${CXX:-c++}}) -MM -E src/common.h -march=native 2>/dev/null || true)"
         },
@@ -20,7 +21,8 @@
             "xcode_settings": {
               "OTHER_CFLAGS": ["-march=native"],
               "OTHER_CXXFLAGS": ["-march=native"],
-            }
+            },
+            "defines": ["YENC_BUILD_NATIVE=1"]
           }]
         ]
       }],
@@ -45,12 +47,9 @@
       ['disable_crcutil!=0', {
         "defines": ["YENC_DISABLE_CRCUTIL=1"]
       }],
-      ['OS!="win" and enable_native_tuning!=0', {
-        "defines": ["YENC_BUILD_NATIVE=1"]
-      }],
       ['OS!="win"', {
         "variables": {
-          "missing_memalign%": "<!(<!(echo ${CC_target:-${CC:-cc}}) -c src/test_alignalloc.c -o /dev/null -Werror 2>/dev/null || echo failed)",
+          "missing_memalign%": "<!(<!(echo ${CXX_target:-${CXX:-c++}}) -c src/test_alignalloc.cc -o /dev/null -Werror 2>/dev/null || echo failed)",
         },
         "conditions": [
           ['missing_memalign!=""', {
@@ -60,7 +59,7 @@
       }]
     ],
     "cflags": ["-Wno-unused-function"],
-    "cxxflags": ["-Wno-unused-function", "-std=c++03", "-D_POSIX_C_SOURCE=200112L"],
+    "cxxflags": ["-Wno-unused-function", "-std=c++03"],
     "xcode_settings": {
       "OTHER_CFLAGS": ["-Wno-unused-function"],
       "OTHER_CXXFLAGS": ["-Wno-unused-function"]

package/index.js

@@ -3,6 +3,7 @@
 var y = require('./build/Release/yencode.node');
 
 var toBuffer = Buffer.alloc ? Buffer.from : Buffer;
+var allocBuffer = (Buffer.allocUnsafe || Buffer);
 var bufferSlice = Buffer.prototype.readBigInt64BE ? Buffer.prototype.subarray : Buffer.prototype.slice;
 
 var nl = toBuffer([13, 10]);
@@ -125,9 +126,17 @@ var propIsNotValidNumber = function(prop) {
 module.exports = {
 	encoding: 'utf8',
 	
-	encode: y.encode,
+	encode: y.encode || function(data, line_size, column_offset) {
+		var buffer = allocBuffer(module.exports.maxSize(data.length, line_size));
+		var length = y.encodeTo(data, buffer, line_size, column_offset);
+		return bufferSlice.call(buffer, 0, length);
+	},
 	encodeTo: y.encodeTo,
-	decode: y.decode,
+	decode: y.decode || function(data, stripDots) {
+		var buffer = allocBuffer(data.length);
+		var length = y.decodeTo(data, buffer, stripDots);
+		return bufferSlice.call(buffer, 0, length);
+	},
 	decodeTo: y.decodeTo,
 	
 	decodeChunk: function(data, output, prev) {
@@ -157,7 +166,13 @@ module.exports = {
 			}
 			if(state < 0) state = decodePrev.indexOf('');
 		}
-		var ret = y.decodeIncr(data, state, output);
+		var ret;
+		if(!output && !y.decode) {
+			output = allocBuffer(data.length);
+			ret = y.decodeIncr(data, state, output);
+			ret.output = bufferSlice.call(output, 0, ret.written);
+		} else
+			ret = y.decodeIncr(data, state, output);
 		
 		return {
 			read: ret.read,
@@ -199,13 +214,17 @@ module.exports = {
 		if(!Buffer.isBuffer(data)) data = toBuffer(data);
 		
 		filename = toBuffer(filename.replace(RE_BADCHAR, '').substring(0, 256), exports.encoding);
-		var e = encodeCrc(data, line_size);
-		return Buffer.concat([
-			toBuffer('=ybegin line='+line_size+' size='+data.length+' name='),
-			filename, nl,
-			e.output,
-			toBuffer('\r\n=yend size='+data.length+' crc32=' + e.crc32.toString('hex'))
-		]);
+		
+		var yBegin = '=ybegin line='+line_size+' size='+data.length+' name=';
+		var yEnd = '\r\n=yend size='+data.length+' crc32=';
+		var ret = allocBuffer(yBegin.length + filename.length + 2 + module.exports.maxSize(data.length, line_size) + yEnd.length + 8);
+		
+		var writePos = ret.write(yBegin);
+		writePos += filename.copy(ret, writePos);
+		writePos += nl.copy(ret, writePos);
+		writePos += y.encodeTo(data, bufferSlice.call(ret, writePos), line_size);
+		writePos += ret.write(yEnd + y.crc32(data).toString('hex'), writePos);
+		return bufferSlice.call(ret, 0, writePos);
 	},
 	multi_post: function(filename, size, parts, line_size) {
 		return new YEncoder(filename, size, parts, line_size);
@@ -325,7 +344,7 @@ module.exports = {
 			warnings.push(DecoderWarning('size_mismatch', 'Size specified for part exceeds size specified for whole file'));
 		
 		if(ret.dataStart) {
-			ret.data = y.decode(bufferSlice.call(data, ret.dataStart, ret.dataEnd), !!isRaw);
+			ret.data = module.exports.decode(bufferSlice.call(data, ret.dataStart, ret.dataEnd), !!isRaw);
 			ret.crc32 = y.crc32(ret.data);
 			var hexCrc = ret.crc32.toString('hex');
 			
@@ -403,16 +422,19 @@ YEncoder.prototype = {
 			this.yInfo = null;
 		}
 		
-		var ret = Buffer.concat([
-			toBuffer('=ybegin part='+this.part),
-			yInfo,
-			toBuffer('=ypart begin='+( this.pos+1 )+' end='+end+'\r\n'),
-			y.encode(data, this.line_size),
-			toBuffer('\r\n=yend size='+data.length+' part='+this.part+' pcrc32='+crc.toString('hex')+fullCrc)
-		]);
+		var yBegin = '=ybegin part='+this.part;
+		var yPart = '=ypart begin='+( this.pos+1 )+' end='+end+'\r\n';
+		var yEnd = '\r\n=yend size='+data.length+' part='+this.part+' pcrc32='+crc.toString('hex')+fullCrc;
+		var ret = allocBuffer(yBegin.length + yInfo.length + yPart.length + module.exports.maxSize(data.length, this.line_size) + yEnd.length);
+		
+		var writePos = ret.write(yBegin);
+		writePos += yInfo.copy(ret, writePos);
+		writePos += ret.write(yPart, writePos);
+		writePos += y.encodeTo(data, bufferSlice.call(ret, writePos), this.line_size);
+		writePos += ret.write(yEnd, writePos);
 		
 		this.pos = end;
-		return ret;
+		return bufferSlice.call(ret, 0, writePos);
 	},
 	
 	_encodeSingle: function(data) {
@@ -428,10 +450,12 @@ YEncoder.prototype = {
 		
 		var yInfo = this.yInfo;
 		this.yInfo = null;
-		return Buffer.concat([
-			yInfo,
-			y.encode(data, this.line_size),
-			toBuffer('\r\n=yend size='+data.length+' crc32=' + this.crc.toString('hex'))
-		]);
+		
+		var tail = '\r\n=yend size='+data.length+' crc32=' + this.crc.toString('hex');
+		var ret = allocBuffer(yInfo.length + module.exports.maxSize(data.length, this.line_size) + tail.length);
+		var writePos = yInfo.copy(ret);
+		writePos += y.encodeTo(data, bufferSlice.call(ret, writePos), this.line_size);
+		writePos += ret.write(tail, writePos);
+		return bufferSlice.call(ret, 0, writePos);
 	}
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yencode",
-  "version": "1.2.1",
+  "version": "1.2.4",
   "description": "SIMD accelerated yEnc encoder/decoder and CRC32 calculator",
   "keywords": [
     "yenc",

package/src/common.h

@@ -40,6 +40,11 @@
 	// MSVC doesn't support C11 aligned_alloc: https://stackoverflow.com/a/62963007
 	#define ALIGN_ALLOC(buf, len, align) *(void**)&(buf) = _aligned_malloc((len), align)
 	#define ALIGN_FREE _aligned_free
+#elif defined(__ANDROID__)
+	// Android NDK may have broken std::aligned_alloc [https://github.com/android/ndk/issues/1339]
+	#include <malloc.h>
+	#define ALIGN_ALLOC(buf, len, align) *(void**)&(buf) = memalign(align, (len))
+	#define ALIGN_FREE free
 #elif defined(_ISOC11_SOURCE)
 	// C11 method
 	// len needs to be a multiple of alignment, although it sometimes works if it isn't...

package/src/crc.cc

@@ -198,15 +198,13 @@ namespace RapidYenc {
 # include <Windows.h>
 #endif
 #ifdef PLATFORM_ARM
-# ifdef __ANDROID__
-#  include <cpu-features.h>
-# elif defined(__APPLE__)
+# if defined(__APPLE__)
 #  include <sys/types.h>
 #  include <sys/sysctl.h>
 # elif defined(__has_include)
 #  if __has_include(<sys/auxv.h>)
 #   include <sys/auxv.h>
-#   ifdef __FreeBSD__
+#   if defined(__FreeBSD__) || defined(__OpenBSD__)
 static unsigned long getauxval(unsigned long cap) {
 	unsigned long ret;
 	elf_aux_info(cap, &ret, sizeof(ret));
@@ -217,6 +215,9 @@ static unsigned long getauxval(unsigned long cap) {
 #    include <asm/hwcap.h>
 #   endif
 #  endif
+#  if defined(__ANDROID__) && __has_include(<cpu-features.h>)
+#   include <cpu-features.h>
+#  endif
 # endif
 #endif
 #if defined(__riscv) && defined(__has_include)

package/src/crc.h

@@ -23,7 +23,7 @@ static inline int crc32_isa_level() {
 
 // computes `n % 0xffffffff` (well, almost), using some bit-hacks
 static inline uint32_t crc32_powmod(uint64_t n) {
-#ifdef __GNUC__
+#if defined(__GNUC__) && (__GNUC__ >= 5 || (defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ > 3))))
 	unsigned res;
 	unsigned carry = __builtin_uadd_overflow(n >> 32, n, &res);
 	res += carry;

package/src/crc_arm_pmull.cc

@@ -207,7 +207,7 @@ static uint32_t crc32_shift_pmull(uint32_t crc1, uint32_t n) {
 void RapidYenc::crc_pmull_set_funcs() {
 	_crc32_multiply = &crc32_multiply_pmull;
 	_crc32_shift = &crc32_shift_pmull;
-	_crc32_isa &= ISA_FEATURE_PMULL;
+	_crc32_isa |= ISA_FEATURE_PMULL;
 }
 
 #else

package/src/decoder_rvv.cc

@@ -209,7 +209,8 @@ HEDLEY_ALWAYS_INLINE void do_decode_rvv(const uint8_t* src, long& len, unsigned
 			if(LIKELIHOOD(0.0001, RV(vcpop_m_b4)(RV(vmandn_mm_b4)(cmpEqShift1, cmp, vl2), vl2) != 0)) {
 				// replicate fix_eqMask, but in vector form
 				vbool4_t groupStart = RV(vmandn_mm_b4)(cmpEq, cmpEqShift1, vl2);
-				vbool4_t evenBits = RV_MASK_CAST(4, 8, RV(vmv_v_x_u8m1)(0x55, vl2));
+				vuint8m1_t evenBitsV = RV(vmv_v_x_u8m1)(0x55, vl2);
+				vbool4_t evenBits = RV_MASK_CAST(4, 8, evenBitsV);
 				vbool4_t evenStart = RV(vmand_mm_b4)(groupStart, evenBits, vl2);
 				
 				// compute `cmpEq + evenStart` to obtain oddGroups

package/src/encoder_avx_base.h

@@ -217,12 +217,12 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				// duplicate halves
 				data1A = _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1);
 				data1B = _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1);
-#if defined(__tune_znver2__) || defined(__tune_znver3__) || defined(__tune_znver4__)
-				data2A = _mm256_permute2x128_si256(dataA, dataA, 0x11);
-				data2B = _mm256_permute2x128_si256(dataB, dataB, 0x11);
-#else
+#if defined(__tune_znver1__) || defined(__tune_bdver4__)
 				data2A = _mm256_permute4x64_epi64(dataA, 0xee);
 				data2B = _mm256_permute4x64_epi64(dataB, 0xee);
+#else
+				data2A = _mm256_permute2x128_si256(dataA, dataA, 0x11);
+				data2B = _mm256_permute2x128_si256(dataB, dataB, 0x11);
 #endif
 				
 				shuf1A = _mm256_load_si256(lookupsAVX2->shufExpand + m1);

package/src/encoder_sse_base.h

@@ -351,7 +351,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_sse(int line_size, int* colOffset, const uin
 #if defined(__POPCNT__) && !defined(__tune_btver1__)
 				if(use_isa & ISA_FEATURE_POPCNT) {
 					shuf2Len = popcnt32(maskA) + 16;
-# if defined(__tune_znver4__) || defined(__tune_znver3__) || defined(__tune_znver2__) || defined(__tune_znver1__) || defined(__tune_btver2__)
+# if defined(__tune_znver6__) || defined(__tune_znver5__) || defined(__tune_znver4__) || defined(__tune_znver3__) || defined(__tune_znver2__) || defined(__tune_znver1__) || defined(__tune_btver2__)
 					shuf1Len = popcnt32(m1) + 8;
 					shuf3Len = popcnt32(m3) + shuf2Len + 8;
 # else

package/src/platform.cc

@@ -1,8 +1,6 @@
 #include "common.h"
 #ifdef PLATFORM_ARM
-# ifdef __ANDROID__
-#  include <cpu-features.h>
-# elif defined(_WIN32)
+# if defined(_WIN32)
 #  define WIN32_LEAN_AND_MEAN
 #  define NOMINMAX
 #  include <Windows.h>
@@ -16,10 +14,13 @@
 #    include <asm/hwcap.h>
 #   endif
 #  endif
+#  if defined(__ANDROID__) && __has_include(<cpu-features.h>)
+#   include <cpu-features.h>
+#  endif
 # endif
 bool RapidYenc::cpu_supports_neon() {
 # if defined(AT_HWCAP)
-#  ifdef __FreeBSD__
+#  if defined(__FreeBSD__) || defined(__OpenBSD__)
 	unsigned long supported;
 	elf_aux_info(AT_HWCAP, &supported, sizeof(supported));
 #   ifdef __aarch64__
@@ -139,11 +140,7 @@ int RapidYenc::cpu_supports_isa() {
 							if(cpuInfo2[3] & 0x80000) {
 								_cpuidX(cpuInfo2, 0x24, 0);
 								if((cpuInfo2[1] & 0xff) >= 1 && ( // minimum AVX10.1
-#ifdef YENC_DISABLE_AVX256
-									cpuInfo2[1] & 0x10000 // AVX10/128
-#else
-									cpuInfo2[1] & 0x20000 // AVX10/256
-#endif
+									cpuInfo2[1] & 0x20000 // AVX10/256 (AVX10/128 is now invalid)
 								)) {
 									if(cpuInfo2[1] & 0x40000) ret |= ISA_FEATURE_EVEX512;
 									return ret | ISA_LEVEL_VBMI2;
@@ -204,7 +201,7 @@ int RapidYenc::cpu_supports_crc_isa() {
 bool RapidYenc::cpu_supports_rvv() {
 # if defined(AT_HWCAP)
 	unsigned long ret;
-#  ifdef __FreeBSD__
+#  if defined(__FreeBSD__) || defined(__OpenBSD__)
 	elf_aux_info(AT_HWCAP, &ret, sizeof(ret));
 #  else
 	ret = getauxval(AT_HWCAP);

package/src/yencode.cc

@@ -13,6 +13,11 @@
 using namespace v8;
 using namespace RapidYenc;
 
+#ifdef V8_ENABLE_SANDBOX
+// V8 Memory Cage is enabled - no external buffers allowed
+# define YENC_NO_EXTERNAL_BUFFER 1
+#endif
+
 static void free_buffer(char* data, void* _size) {
 #if !NODE_VERSION_AT_LEAST(0, 11, 0)
 	int size = (int)(size_t)_size;
@@ -101,6 +106,7 @@ static inline size_t YENC_MAX_SIZE(size_t len, size_t line_size) {
 #endif
 
 
+#ifndef YENC_NO_EXTERNAL_BUFFER
 // encode(str, line_size, col)
 FUNC(Encode) {
 	FUNC_START;
@@ -135,6 +141,7 @@ FUNC(Encode) {
 	MARK_EXT_MEM(len);
 	RETURN_VAL( NEW_BUFFER((char*)result, len, free_buffer, (void*)len) );
 }
+#endif
 
 FUNC(EncodeTo) {
 	FUNC_START;
@@ -205,6 +212,9 @@ FUNC(EncodeIncr) {
 			}
 		}
 	}
+#ifdef YENC_NO_EXTERNAL_BUFFER
+	if(allocResult) RETURN_ERROR("Destination buffer must be supplied as buffer allocation isn't enabled in this build");
+#endif
 	
 	Local<Object> ret = NEW_OBJECT;
 	
@@ -215,24 +225,29 @@ FUNC(EncodeIncr) {
 		RETURN_VAL( ret );
 	}
 	
+#ifndef YENC_NO_EXTERNAL_BUFFER
 	if(allocResult) {
 		// allocate enough memory to handle worst case requirements
 		size_t dest_len = YENC_MAX_SIZE(arg_len, line_size);
 		result = (unsigned char*) malloc(dest_len);
 	}
+#endif
 	
 	size_t len = encode(line_size, &col, (const unsigned char*)node::Buffer::Data(args[0]), result, arg_len, false);
 	
 	SET_OBJ(ret, "written", Integer::New(ISOLATE len));
+#ifndef YENC_NO_EXTERNAL_BUFFER
 	if(allocResult) {
 		result = (unsigned char*)realloc(result, len);
 		SET_OBJ(ret, "output", NEW_BUFFER((char*)result, len, free_buffer, (void*)len));
 		MARK_EXT_MEM(len);
 	}
+#endif
 	SET_OBJ(ret, "col", Integer::New(ISOLATE col));
 	RETURN_VAL( ret );
 }
 
+#ifndef YENC_NO_EXTERNAL_BUFFER
 FUNC(Decode) {
 	FUNC_START;
 	
@@ -253,6 +268,7 @@ FUNC(Decode) {
 	MARK_EXT_MEM(len);
 	RETURN_VAL( NEW_BUFFER((char*)result, len, free_buffer, (void*)len) );
 }
+#endif
 
 FUNC(DecodeTo) {
 	FUNC_START;
@@ -301,6 +317,10 @@ FUNC(DecodeIncr) {
 		}
 	}
 	
+#ifdef YENC_NO_EXTERNAL_BUFFER
+	if(allocResult) RETURN_ERROR("Destination buffer must be supplied as buffer allocation isn't enabled in this build");
+#endif
+	
 	const unsigned char* src = (const unsigned char*)node::Buffer::Data(args[0]);
 	const unsigned char* sp = src;
 #ifdef DBG_ALIGN_SOURCE
@@ -309,11 +329,15 @@ FUNC(DecodeIncr) {
 	sp = (const unsigned char*)newSrc;
 #endif
 	
+#ifndef YENC_NO_EXTERNAL_BUFFER
 	if(allocResult) result = (unsigned char*) malloc(arg_len);
+#endif
 	unsigned char* dp = result;
 	YencDecoderEnd ended = RapidYenc::decode_end((const void**)&sp, (void**)&dp, arg_len, &state);
 	size_t len = dp - result;
+#ifndef YENC_NO_EXTERNAL_BUFFER
 	if(allocResult) result = (unsigned char*)realloc(result, len);
+#endif
 	
 #ifdef DBG_ALIGN_SOURCE
 	free(newSrc);
@@ -322,10 +346,12 @@ FUNC(DecodeIncr) {
 	Local<Object> ret = NEW_OBJECT;
 	SET_OBJ(ret, "read", Integer::New(ISOLATE sp - src));
 	SET_OBJ(ret, "written", Integer::New(ISOLATE len));
+#ifndef YENC_NO_EXTERNAL_BUFFER
 	if(allocResult) {
 		SET_OBJ(ret, "output", NEW_BUFFER((char*)result, len, free_buffer, (void*)len));
 		MARK_EXT_MEM(len);
 	}
+#endif
 	SET_OBJ(ret, "ended", Integer::New(ISOLATE (int)ended));
 	SET_OBJ(ret, "state", Integer::New(ISOLATE state));
 	RETURN_VAL( ret );
@@ -468,10 +494,12 @@ void yencode_init(
 )
 #endif
 {
+#ifndef YENC_NO_EXTERNAL_BUFFER
 	NODE_SET_METHOD(exports, "encode", Encode);
+	NODE_SET_METHOD(exports, "decode", Decode);
+#endif
 	NODE_SET_METHOD(exports, "encodeTo", EncodeTo);
 	NODE_SET_METHOD(exports, "encodeIncr", EncodeIncr);
-	NODE_SET_METHOD(exports, "decode", Decode);
 	NODE_SET_METHOD(exports, "decodeTo", DecodeTo);
 	NODE_SET_METHOD(exports, "decodeIncr", DecodeIncr);
 	NODE_SET_METHOD(exports, "crc32", CRC32);