visaljs 0.0.2 → 0.0.3

package/runtime/third_party/boringssl/gen/bcm/aes-gcm-avx512-x86_64-apple.S

@@ -0,0 +1,1252 @@
+// This file is generated from a similarly-named Perl script in the BoringSSL
+// source tree. Do not edit by hand.
+
+#include <openssl/asm_base.h>
+
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__)
+.section	__DATA,__const
+
+.p2align	6
+aes_gcm_avx512_constants:
+
+// A shuffle mask that reflects the bytes of 16-byte blocks
+L$bswap_mask:
+.quad	0x08090a0b0c0d0e0f, 0x0001020304050607
+
+// This is the GHASH reducing polynomial without its constant term, i.e.
+// x^128 + x^7 + x^2 + x, represented using the backwards mapping
+// between bits and polynomial coefficients.
+// 
+// Alternatively, it can be interpreted as the naturally-ordered
+// representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the
+// "reversed" GHASH reducing polynomial without its x^128 term.
+L$gfpoly:
+.quad	1, 0xc200000000000000
+
+// Same as above, but with the (1 << 64) bit set.
+L$gfpoly_and_internal_carrybit:
+.quad	1, 0xc200000000000001
+
+// Values needed to prepare the initial vector of counter blocks.
+L$ctr_pattern:
+.quad	0, 0
+.quad	1, 0
+.quad	2, 0
+.quad	3, 0
+
+// The number of AES blocks per vector, as a 128-bit value.
+L$inc_4blocks:
+.quad	4, 0
+
+.text	
+.globl	_gcm_init_vpclmulqdq_avx512
+.private_extern _gcm_init_vpclmulqdq_avx512
+
+.p2align	5
+_gcm_init_vpclmulqdq_avx512:
+
+
+_CET_ENDBR
+// Get pointer to lowest set of key powers (located at end of array).
+	leaq	256-64(%rdi),%r8
+
+// Load the byte-reflected hash subkey.  BoringSSL provides it in
+// byte-reflected form except the two halves are in the wrong order.
+	vpshufd	$0x4e,(%rsi),%xmm3
+
+// Finish preprocessing the first key power, H^1.  Since this GHASH
+// implementation operates directly on values with the backwards bit
+// order specified by the GCM standard, it's necessary to preprocess the
+// raw key as follows.  First, reflect its bytes.  Second, multiply it
+// by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards
+// interpretation of polynomial coefficients), which can also be
+// interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121
+// + 1 using the alternative, natural interpretation of polynomial
+// coefficients.  For details, see the comment above _ghash_mul.
+// 
+// Either way, for the multiplication the concrete operation performed
+// is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2
+// << 120) | 1 if a 1 bit was carried out.  However, there's no 128-bit
+// wide shift instruction, so instead double each of the two 64-bit
+// halves and incorporate the internal carry bit into the value XOR'd.
+	vpshufd	$0xd3,%xmm3,%xmm0
+	vpsrad	$31,%xmm0,%xmm0
+	vpaddq	%xmm3,%xmm3,%xmm3
+// H_CUR_XMM ^= TMP0_XMM & gfpoly_and_internal_carrybit
+	vpternlogd	$0x78,L$gfpoly_and_internal_carrybit(%rip),%xmm0,%xmm3
+
+// Load the gfpoly constant.
+	vbroadcasti32x4	L$gfpoly(%rip),%zmm5
+
+// Square H^1 to get H^2.
+// 
+// Note that as with H^1, all higher key powers also need an extra
+// factor of x^-1 (or x using the natural interpretation).  Nothing
+// special needs to be done to make this happen, though: H^1 * H^1 would
+// end up with two factors of x^-1, but the multiplication consumes one.
+// So the product H^2 ends up with the desired one factor of x^-1.
+	vpclmulqdq	$0x00,%xmm3,%xmm3,%xmm0  // LO = a_L * a_L
+	vpclmulqdq	$0x11,%xmm3,%xmm3,%xmm4  // HI = a_H * a_H
+	vpclmulqdq	$0x01,%xmm0,%xmm5,%xmm1  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%xmm0,%xmm0  // Swap halves of LO
+	vpxor	%xmm0,%xmm1,%xmm1  // Fold LO into MI
+	vpclmulqdq	$0x01,%xmm1,%xmm5,%xmm0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%xmm1,%xmm1  // Swap halves of MI
+	vpternlogd	$0x96,%xmm0,%xmm1,%xmm4  // Fold MI into HI
+
+
+// Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2].
+	vinserti128	$1,%xmm3,%ymm4,%ymm3
+	vinserti128	$1,%xmm4,%ymm4,%ymm4
+
+// Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4].
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm0  // LO = a_L * b_L
+	vpclmulqdq	$0x01,%ymm4,%ymm3,%ymm1  // MI_0 = a_L * b_H
+	vpclmulqdq	$0x10,%ymm4,%ymm3,%ymm2  // MI_1 = a_H * b_L
+	vpxord	%ymm2,%ymm1,%ymm1  // MI = MI_0 + MI_1
+	vpclmulqdq	$0x01,%ymm0,%ymm5,%ymm2  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%ymm0,%ymm0  // Swap halves of LO
+	vpternlogd	$0x96,%ymm2,%ymm0,%ymm1  // Fold LO into MI
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm4  // HI = a_H * b_H
+	vpclmulqdq	$0x01,%ymm1,%ymm5,%ymm0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%ymm1,%ymm1  // Swap halves of MI
+	vpternlogd	$0x96,%ymm0,%ymm1,%ymm4  // Fold MI into HI
+
+	vinserti64x4	$1,%ymm3,%zmm4,%zmm3
+	vshufi64x2	$0,%zmm4,%zmm4,%zmm4
+
+// Store the lowest set of key powers.
+	vmovdqu8	%zmm3,(%r8)
+
+// Compute and store the remaining key powers.
+// Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by
+// [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)].
+	movl	$3,%eax
+L$precompute_next:
+	subq	$64,%r8
+	vpclmulqdq	$0x00,%zmm4,%zmm3,%zmm0  // LO = a_L * b_L
+	vpclmulqdq	$0x01,%zmm4,%zmm3,%zmm1  // MI_0 = a_L * b_H
+	vpclmulqdq	$0x10,%zmm4,%zmm3,%zmm2  // MI_1 = a_H * b_L
+	vpxord	%zmm2,%zmm1,%zmm1  // MI = MI_0 + MI_1
+	vpclmulqdq	$0x01,%zmm0,%zmm5,%zmm2  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%zmm0,%zmm0  // Swap halves of LO
+	vpternlogd	$0x96,%zmm2,%zmm0,%zmm1  // Fold LO into MI
+	vpclmulqdq	$0x11,%zmm4,%zmm3,%zmm3  // HI = a_H * b_H
+	vpclmulqdq	$0x01,%zmm1,%zmm5,%zmm0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%zmm1,%zmm1  // Swap halves of MI
+	vpternlogd	$0x96,%zmm0,%zmm1,%zmm3  // Fold MI into HI
+
+	vmovdqu8	%zmm3,(%r8)
+	decl	%eax
+	jnz	L$precompute_next
+
+	vzeroupper  // This is needed after using ymm or zmm registers.
+	ret
+
+
+
+.globl	_gcm_gmult_vpclmulqdq_avx512
+.private_extern _gcm_gmult_vpclmulqdq_avx512
+
+.p2align	5
+.alt_entry _gcm_gmult_vpclmulqdq_avx512
+_gcm_gmult_vpclmulqdq_avx512:
+
+
+_CET_ENDBR
+
+
+
+	vmovdqu	(%rdi),%xmm0
+	vmovdqu	L$bswap_mask(%rip),%xmm1
+	vmovdqu	256-16(%rsi),%xmm2
+	vmovdqu	L$gfpoly(%rip),%xmm3
+	vpshufb	%xmm1,%xmm0,%xmm0
+
+	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm4  // LO = a_L * b_L
+	vpclmulqdq	$0x01,%xmm2,%xmm0,%xmm5  // MI_0 = a_L * b_H
+	vpclmulqdq	$0x10,%xmm2,%xmm0,%xmm6  // MI_1 = a_H * b_L
+	vpxord	%xmm6,%xmm5,%xmm5  // MI = MI_0 + MI_1
+	vpclmulqdq	$0x01,%xmm4,%xmm3,%xmm6  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%xmm4,%xmm4  // Swap halves of LO
+	vpternlogd	$0x96,%xmm6,%xmm4,%xmm5  // Fold LO into MI
+	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm0  // HI = a_H * b_H
+	vpclmulqdq	$0x01,%xmm5,%xmm3,%xmm4  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%xmm5,%xmm5  // Swap halves of MI
+	vpternlogd	$0x96,%xmm4,%xmm5,%xmm0  // Fold MI into HI
+
+
+	vpshufb	%xmm1,%xmm0,%xmm0
+	vmovdqu	%xmm0,(%rdi)
+
+// No need for vzeroupper, since only xmm registers were used.
+	ret
+
+
+
+.globl	_gcm_ghash_vpclmulqdq_avx512
+.private_extern _gcm_ghash_vpclmulqdq_avx512
+
+.p2align	5
+.alt_entry _gcm_ghash_vpclmulqdq_avx512
+_gcm_ghash_vpclmulqdq_avx512:
+
+
+_CET_ENDBR
+
+
+
+// Load the bswap_mask and gfpoly constants.  Since AADLEN is usually small,
+// usually only 128-bit vectors will be used.  So as an optimization, don't
+// broadcast these constants to all 128-bit lanes quite yet.
+	vmovdqu	L$bswap_mask(%rip),%xmm4
+	vmovdqu	L$gfpoly(%rip),%xmm10
+
+// Load the GHASH accumulator.
+	vmovdqu	(%rdi),%xmm5
+	vpshufb	%xmm4,%xmm5,%xmm5
+
+// Optimize for AADLEN < 64 by checking for AADLEN < 64 before AADLEN < 256.
+	cmpq	$64,%rcx
+	jb	L$aad_blockbyblock
+
+// AADLEN >= 64, so we'll operate on full vectors.  Broadcast bswap_mask and
+// gfpoly to all 128-bit lanes.
+	vshufi64x2	$0,%zmm4,%zmm4,%zmm4
+	vshufi64x2	$0,%zmm10,%zmm10,%zmm10
+
+// Load the lowest set of key powers.
+	vmovdqu8	256-64(%rsi),%zmm9
+
+	cmpq	$256,%rcx
+	jb	L$aad_loop_1x
+
+// AADLEN >= 256.  Load the higher key powers.
+	vmovdqu8	256-256(%rsi),%zmm6
+	vmovdqu8	256-192(%rsi),%zmm7
+	vmovdqu8	256-128(%rsi),%zmm8
+
+// Update GHASH with 256 bytes of AAD at a time.
+L$aad_loop_4x:
+	vmovdqu8	0(%rdx),%zmm0
+	vmovdqu8	64(%rdx),%zmm1
+	vmovdqu8	128(%rdx),%zmm2
+	vmovdqu8	192(%rdx),%zmm3
+	vpshufb	%zmm4,%zmm0,%zmm0
+	vpxord	%zmm5,%zmm0,%zmm0
+	vpshufb	%zmm4,%zmm1,%zmm1
+	vpshufb	%zmm4,%zmm2,%zmm2
+	vpshufb	%zmm4,%zmm3,%zmm3
+	vpclmulqdq	$0x00,%zmm6,%zmm0,%zmm5  // LO_0
+	vpclmulqdq	$0x00,%zmm7,%zmm1,%zmm11  // LO_1
+	vpclmulqdq	$0x00,%zmm8,%zmm2,%zmm12  // LO_2
+	vpxord	%zmm11,%zmm5,%zmm5  // sum(LO_{1,0})
+	vpclmulqdq	$0x00,%zmm9,%zmm3,%zmm13  // LO_3
+	vpternlogd	$0x96,%zmm13,%zmm12,%zmm5  // LO = sum(LO_{3,2,1,0})
+	vpclmulqdq	$0x01,%zmm6,%zmm0,%zmm11  // MI_0
+	vpclmulqdq	$0x01,%zmm7,%zmm1,%zmm12  // MI_1
+	vpclmulqdq	$0x01,%zmm8,%zmm2,%zmm13  // MI_2
+	vpternlogd	$0x96,%zmm13,%zmm12,%zmm11  // sum(MI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm9,%zmm3,%zmm12  // MI_3
+	vpclmulqdq	$0x10,%zmm6,%zmm0,%zmm13  // MI_4
+	vpternlogd	$0x96,%zmm13,%zmm12,%zmm11  // sum(MI_{4,3,2,1,0})
+	vpclmulqdq	$0x10,%zmm7,%zmm1,%zmm12  // MI_5
+	vpclmulqdq	$0x10,%zmm8,%zmm2,%zmm13  // MI_6
+	vpternlogd	$0x96,%zmm13,%zmm12,%zmm11  // sum(MI_{6,5,4,3,2,1,0})
+	vpclmulqdq	$0x01,%zmm5,%zmm10,%zmm13  // LO_L*(x^63 + x^62 + x^57)
+	vpclmulqdq	$0x10,%zmm9,%zmm3,%zmm12  // MI_7
+	vpxord	%zmm12,%zmm11,%zmm11  // MI = sum(MI_{7,6,5,4,3,2,1,0})
+	vpshufd	$0x4e,%zmm5,%zmm5  // Swap halves of LO
+	vpclmulqdq	$0x11,%zmm6,%zmm0,%zmm0  // HI_0
+	vpclmulqdq	$0x11,%zmm7,%zmm1,%zmm1  // HI_1
+	vpclmulqdq	$0x11,%zmm8,%zmm2,%zmm2  // HI_2
+	vpternlogd	$0x96,%zmm13,%zmm5,%zmm11  // Fold LO into MI
+	vpclmulqdq	$0x11,%zmm9,%zmm3,%zmm3  // HI_3
+	vpternlogd	$0x96,%zmm2,%zmm1,%zmm0  // sum(HI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm11,%zmm10,%zmm12  // MI_L*(x^63 + x^62 + x^57)
+	vpxord	%zmm3,%zmm0,%zmm5  // HI = sum(HI_{3,2,1,0})
+	vpshufd	$0x4e,%zmm11,%zmm11  // Swap halves of MI
+	vpternlogd	$0x96,%zmm12,%zmm11,%zmm5  // Fold MI into HI
+	vextracti32x4	$1,%zmm5,%xmm0
+	vextracti32x4	$2,%zmm5,%xmm1
+	vextracti32x4	$3,%zmm5,%xmm2
+	vpxord	%xmm0,%xmm5,%xmm5
+	vpternlogd	$0x96,%xmm1,%xmm2,%xmm5
+
+	addq	$256,%rdx
+	subq	$256,%rcx
+	cmpq	$256,%rcx
+	jae	L$aad_loop_4x
+
+// Update GHASH with 64 bytes of AAD at a time.
+	cmpq	$64,%rcx
+	jb	L$aad_large_done
+L$aad_loop_1x:
+	vmovdqu8	(%rdx),%zmm0
+	vpshufb	%zmm4,%zmm0,%zmm0
+	vpxord	%zmm0,%zmm5,%zmm5
+	vpclmulqdq	$0x00,%zmm9,%zmm5,%zmm0  // LO = a_L * b_L
+	vpclmulqdq	$0x01,%zmm9,%zmm5,%zmm1  // MI_0 = a_L * b_H
+	vpclmulqdq	$0x10,%zmm9,%zmm5,%zmm2  // MI_1 = a_H * b_L
+	vpxord	%zmm2,%zmm1,%zmm1  // MI = MI_0 + MI_1
+	vpclmulqdq	$0x01,%zmm0,%zmm10,%zmm2  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%zmm0,%zmm0  // Swap halves of LO
+	vpternlogd	$0x96,%zmm2,%zmm0,%zmm1  // Fold LO into MI
+	vpclmulqdq	$0x11,%zmm9,%zmm5,%zmm5  // HI = a_H * b_H
+	vpclmulqdq	$0x01,%zmm1,%zmm10,%zmm0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%zmm1,%zmm1  // Swap halves of MI
+	vpternlogd	$0x96,%zmm0,%zmm1,%zmm5  // Fold MI into HI
+
+	vextracti32x4	$1,%zmm5,%xmm0
+	vextracti32x4	$2,%zmm5,%xmm1
+	vextracti32x4	$3,%zmm5,%xmm2
+	vpxord	%xmm0,%xmm5,%xmm5
+	vpternlogd	$0x96,%xmm1,%xmm2,%xmm5
+
+	addq	$64,%rdx
+	subq	$64,%rcx
+	cmpq	$64,%rcx
+	jae	L$aad_loop_1x
+
+L$aad_large_done:
+
+// GHASH the remaining data 16 bytes at a time, using xmm registers only.
+L$aad_blockbyblock:
+	testq	%rcx,%rcx
+	jz	L$aad_done
+	vmovdqu	256-16(%rsi),%xmm9
+L$aad_loop_blockbyblock:
+	vmovdqu	(%rdx),%xmm0
+	vpshufb	%xmm4,%xmm0,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm9,%xmm5,%xmm0  // LO = a_L * b_L
+	vpclmulqdq	$0x01,%xmm9,%xmm5,%xmm1  // MI_0 = a_L * b_H
+	vpclmulqdq	$0x10,%xmm9,%xmm5,%xmm2  // MI_1 = a_H * b_L
+	vpxord	%xmm2,%xmm1,%xmm1  // MI = MI_0 + MI_1
+	vpclmulqdq	$0x01,%xmm0,%xmm10,%xmm2  // LO_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%xmm0,%xmm0  // Swap halves of LO
+	vpternlogd	$0x96,%xmm2,%xmm0,%xmm1  // Fold LO into MI
+	vpclmulqdq	$0x11,%xmm9,%xmm5,%xmm5  // HI = a_H * b_H
+	vpclmulqdq	$0x01,%xmm1,%xmm10,%xmm0  // MI_L*(x^63 + x^62 + x^57)
+	vpshufd	$0x4e,%xmm1,%xmm1  // Swap halves of MI
+	vpternlogd	$0x96,%xmm0,%xmm1,%xmm5  // Fold MI into HI
+
+	addq	$16,%rdx
+	subq	$16,%rcx
+	jnz	L$aad_loop_blockbyblock
+
+L$aad_done:
+// Store the updated GHASH accumulator back to memory.
+	vpshufb	%xmm4,%xmm5,%xmm5
+	vmovdqu	%xmm5,(%rdi)
+
+	vzeroupper  // This is needed after using ymm or zmm registers.
+	ret
+
+
+
+.globl	_aes_gcm_enc_update_vaes_avx512
+.private_extern _aes_gcm_enc_update_vaes_avx512
+
+.p2align	5
+.alt_entry _aes_gcm_enc_update_vaes_avx512
+_aes_gcm_enc_update_vaes_avx512:
+
+
+_CET_ENDBR
+	pushq	%r12
+
+
+	movq	16(%rsp),%r12  // arg7
+#ifdef BORINGSSL_DISPATCH_TEST
+
+	movb	$1,_BORINGSSL_function_hit+7(%rip)
+#endif
+// Load some constants.
+	vbroadcasti32x4	L$bswap_mask(%rip),%zmm8
+	vbroadcasti32x4	L$gfpoly(%rip),%zmm31
+
+// Load the GHASH accumulator and the starting counter.
+// BoringSSL passes these values in big endian format.
+	vmovdqu	(%r12),%xmm10
+	vpshufb	%xmm8,%xmm10,%xmm10
+	vbroadcasti32x4	(%r8),%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm12
+
+// Load the AES key length in bytes.  BoringSSL stores number of rounds
+// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20.
+	movl	240(%rcx),%r10d
+	leal	-20(,%r10,4),%r10d
+
+// Make RNDKEYLAST_PTR point to the last AES round key.  This is the
+// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
+// respectively.  Then load the zero-th and last round keys.
+	leaq	96(%rcx,%r10,4),%r11
+	vbroadcasti32x4	(%rcx),%zmm13
+	vbroadcasti32x4	(%r11),%zmm14
+
+// Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words.
+	vpaddd	L$ctr_pattern(%rip),%zmm12,%zmm12
+
+// Load 4 into all 128-bit lanes of LE_CTR_INC.
+	vbroadcasti32x4	L$inc_4blocks(%rip),%zmm11
+
+// If there are at least 256 bytes of data, then continue into the loop
+// that processes 256 bytes of data at a time.  Otherwise skip it.
+	cmpq	$256,%rdx
+	jb	L$crypt_loop_4x_done__func1
+
+// Load powers of the hash key.
+	vmovdqu8	256-256(%r9),%zmm27
+	vmovdqu8	256-192(%r9),%zmm28
+	vmovdqu8	256-128(%r9),%zmm29
+	vmovdqu8	256-64(%r9),%zmm30
+// Encrypt the first 4 vectors of plaintext blocks.  Leave the resulting
+// ciphertext in GHASHDATA[0-3] for GHASH.
+// Increment le_ctr four times to generate four vectors of little-endian
+// counter blocks, swap each to big-endian, and store them in aesdata[0-3].
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm1
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm2
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm3
+	vpaddd	%zmm11,%zmm12,%zmm12
+
+// AES "round zero": XOR in the zero-th round key.
+	vpxord	%zmm13,%zmm0,%zmm0
+	vpxord	%zmm13,%zmm1,%zmm1
+	vpxord	%zmm13,%zmm2,%zmm2
+	vpxord	%zmm13,%zmm3,%zmm3
+
+	leaq	16(%rcx),%rax
+L$vaesenc_loop_first_4_vecs__func1:
+	vbroadcasti32x4	(%rax),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	L$vaesenc_loop_first_4_vecs__func1
+	vpxord	0(%rdi),%zmm14,%zmm4
+	vpxord	64(%rdi),%zmm14,%zmm5
+	vpxord	128(%rdi),%zmm14,%zmm6
+	vpxord	192(%rdi),%zmm14,%zmm7
+	vaesenclast	%zmm4,%zmm0,%zmm4
+	vaesenclast	%zmm5,%zmm1,%zmm5
+	vaesenclast	%zmm6,%zmm2,%zmm6
+	vaesenclast	%zmm7,%zmm3,%zmm7
+	vmovdqu8	%zmm4,0(%rsi)
+	vmovdqu8	%zmm5,64(%rsi)
+	vmovdqu8	%zmm6,128(%rsi)
+	vmovdqu8	%zmm7,192(%rsi)
+
+	addq	$256,%rdi
+	addq	$256,%rsi
+	subq	$256,%rdx
+	cmpq	$256,%rdx
+	jb	L$ghash_last_ciphertext_4x__func1
+// Cache as many additional AES round keys as possible.
+	vbroadcasti32x4	-144(%r11),%zmm15
+	vbroadcasti32x4	-128(%r11),%zmm16
+	vbroadcasti32x4	-112(%r11),%zmm17
+	vbroadcasti32x4	-96(%r11),%zmm18
+	vbroadcasti32x4	-80(%r11),%zmm19
+	vbroadcasti32x4	-64(%r11),%zmm20
+	vbroadcasti32x4	-48(%r11),%zmm21
+	vbroadcasti32x4	-32(%r11),%zmm22
+	vbroadcasti32x4	-16(%r11),%zmm23
+
+L$crypt_loop_4x__func1:
+// Start the AES encryption of the counter blocks.
+// Increment le_ctr four times to generate four vectors of little-endian
+// counter blocks, swap each to big-endian, and store them in aesdata[0-3].
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm1
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm2
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm3
+	vpaddd	%zmm11,%zmm12,%zmm12
+
+// AES "round zero": XOR in the zero-th round key.
+	vpxord	%zmm13,%zmm0,%zmm0
+	vpxord	%zmm13,%zmm1,%zmm1
+	vpxord	%zmm13,%zmm2,%zmm2
+	vpxord	%zmm13,%zmm3,%zmm3
+
+	cmpl	$24,%r10d
+	jl	L$aes128__func1
+	je	L$aes192__func1
+// AES-256
+	vbroadcasti32x4	-208(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+	vbroadcasti32x4	-192(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+L$aes192__func1:
+	vbroadcasti32x4	-176(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+	vbroadcasti32x4	-160(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+L$aes128__func1:
+
+// Prefetch the source data 512 bytes ahead into the L1 data cache, to
+// improve performance when the hardware prefetcher is disabled.  Assumes the
+// L1 data cache line size is 64 bytes (de facto standard on x86_64).
+	prefetcht0	512+0(%rdi)
+	prefetcht0	512+64(%rdi)
+	prefetcht0	512+128(%rdi)
+	prefetcht0	512+192(%rdi)
+
+// Finish the AES encryption of the counter blocks in AESDATA[0-3],
+// interleaved with the GHASH update of the ciphertext blocks in
+// GHASHDATA[0-3].
+	vpshufb	%zmm8,%zmm4,%zmm4
+	vpxord	%zmm10,%zmm4,%zmm4
+	vpshufb	%zmm8,%zmm5,%zmm5
+	vpshufb	%zmm8,%zmm6,%zmm6
+
+	vaesenc	%zmm15,%zmm0,%zmm0
+	vaesenc	%zmm15,%zmm1,%zmm1
+	vaesenc	%zmm15,%zmm2,%zmm2
+	vaesenc	%zmm15,%zmm3,%zmm3
+
+	vpshufb	%zmm8,%zmm7,%zmm7
+	vpclmulqdq	$0x00,%zmm27,%zmm4,%zmm10  // LO_0
+	vpclmulqdq	$0x00,%zmm28,%zmm5,%zmm24  // LO_1
+	vpclmulqdq	$0x00,%zmm29,%zmm6,%zmm25  // LO_2
+
+	vaesenc	%zmm16,%zmm0,%zmm0
+	vaesenc	%zmm16,%zmm1,%zmm1
+	vaesenc	%zmm16,%zmm2,%zmm2
+	vaesenc	%zmm16,%zmm3,%zmm3
+
+	vpxord	%zmm24,%zmm10,%zmm10  // sum(LO_{1,0})
+	vpclmulqdq	$0x00,%zmm30,%zmm7,%zmm26  // LO_3
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm10  // LO = sum(LO_{3,2,1,0})
+	vpclmulqdq	$0x01,%zmm27,%zmm4,%zmm24  // MI_0
+
+	vaesenc	%zmm17,%zmm0,%zmm0
+	vaesenc	%zmm17,%zmm1,%zmm1
+	vaesenc	%zmm17,%zmm2,%zmm2
+	vaesenc	%zmm17,%zmm3,%zmm3
+
+	vpclmulqdq	$0x01,%zmm28,%zmm5,%zmm25  // MI_1
+	vpclmulqdq	$0x01,%zmm29,%zmm6,%zmm26  // MI_2
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm30,%zmm7,%zmm25  // MI_3
+
+	vaesenc	%zmm18,%zmm0,%zmm0
+	vaesenc	%zmm18,%zmm1,%zmm1
+	vaesenc	%zmm18,%zmm2,%zmm2
+	vaesenc	%zmm18,%zmm3,%zmm3
+
+	vpclmulqdq	$0x10,%zmm27,%zmm4,%zmm26  // MI_4
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{4,3,2,1,0})
+	vpclmulqdq	$0x10,%zmm28,%zmm5,%zmm25  // MI_5
+	vpclmulqdq	$0x10,%zmm29,%zmm6,%zmm26  // MI_6
+
+	vaesenc	%zmm19,%zmm0,%zmm0
+	vaesenc	%zmm19,%zmm1,%zmm1
+	vaesenc	%zmm19,%zmm2,%zmm2
+	vaesenc	%zmm19,%zmm3,%zmm3
+
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{6,5,4,3,2,1,0})
+	vpclmulqdq	$0x01,%zmm10,%zmm31,%zmm26  // LO_L*(x^63 + x^62 + x^57)
+	vpclmulqdq	$0x10,%zmm30,%zmm7,%zmm25  // MI_7
+	vpxord	%zmm25,%zmm24,%zmm24  // MI = sum(MI_{7,6,5,4,3,2,1,0})
+
+	vaesenc	%zmm20,%zmm0,%zmm0
+	vaesenc	%zmm20,%zmm1,%zmm1
+	vaesenc	%zmm20,%zmm2,%zmm2
+	vaesenc	%zmm20,%zmm3,%zmm3
+
+	vpshufd	$0x4e,%zmm10,%zmm10  // Swap halves of LO
+	vpclmulqdq	$0x11,%zmm27,%zmm4,%zmm4  // HI_0
+	vpclmulqdq	$0x11,%zmm28,%zmm5,%zmm5  // HI_1
+	vpclmulqdq	$0x11,%zmm29,%zmm6,%zmm6  // HI_2
+
+	vaesenc	%zmm21,%zmm0,%zmm0
+	vaesenc	%zmm21,%zmm1,%zmm1
+	vaesenc	%zmm21,%zmm2,%zmm2
+	vaesenc	%zmm21,%zmm3,%zmm3
+
+	vpternlogd	$0x96,%zmm26,%zmm10,%zmm24  // Fold LO into MI
+	vpclmulqdq	$0x11,%zmm30,%zmm7,%zmm7  // HI_3
+	vpternlogd	$0x96,%zmm6,%zmm5,%zmm4  // sum(HI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm24,%zmm31,%zmm25  // MI_L*(x^63 + x^62 + x^57)
+
+	vaesenc	%zmm22,%zmm0,%zmm0
+	vaesenc	%zmm22,%zmm1,%zmm1
+	vaesenc	%zmm22,%zmm2,%zmm2
+	vaesenc	%zmm22,%zmm3,%zmm3
+
+	vpxord	%zmm7,%zmm4,%zmm10  // HI = sum(HI_{3,2,1,0})
+	vpshufd	$0x4e,%zmm24,%zmm24  // Swap halves of MI
+	vpternlogd	$0x96,%zmm25,%zmm24,%zmm10  // Fold MI into HI
+
+	vaesenc	%zmm23,%zmm0,%zmm0
+	vaesenc	%zmm23,%zmm1,%zmm1
+	vaesenc	%zmm23,%zmm2,%zmm2
+	vaesenc	%zmm23,%zmm3,%zmm3
+
+
+	vextracti32x4	$1,%zmm10,%xmm4
+	vextracti32x4	$2,%zmm10,%xmm5
+	vextracti32x4	$3,%zmm10,%xmm6
+	vpxord	%xmm4,%xmm10,%xmm10
+	vpternlogd	$0x96,%xmm5,%xmm6,%xmm10
+
+	vpxord	0(%rdi),%zmm14,%zmm4
+	vpxord	64(%rdi),%zmm14,%zmm5
+	vpxord	128(%rdi),%zmm14,%zmm6
+	vpxord	192(%rdi),%zmm14,%zmm7
+	vaesenclast	%zmm4,%zmm0,%zmm4
+	vaesenclast	%zmm5,%zmm1,%zmm5
+	vaesenclast	%zmm6,%zmm2,%zmm6
+	vaesenclast	%zmm7,%zmm3,%zmm7
+	vmovdqu8	%zmm4,0(%rsi)
+	vmovdqu8	%zmm5,64(%rsi)
+	vmovdqu8	%zmm6,128(%rsi)
+	vmovdqu8	%zmm7,192(%rsi)
+
+	addq	$256,%rdi
+	addq	$256,%rsi
+	subq	$256,%rdx
+	cmpq	$256,%rdx
+	jae	L$crypt_loop_4x__func1
+L$ghash_last_ciphertext_4x__func1:
+	vpshufb	%zmm8,%zmm4,%zmm4
+	vpxord	%zmm10,%zmm4,%zmm4
+	vpshufb	%zmm8,%zmm5,%zmm5
+	vpshufb	%zmm8,%zmm6,%zmm6
+	vpshufb	%zmm8,%zmm7,%zmm7
+	vpclmulqdq	$0x00,%zmm27,%zmm4,%zmm10  // LO_0
+	vpclmulqdq	$0x00,%zmm28,%zmm5,%zmm24  // LO_1
+	vpclmulqdq	$0x00,%zmm29,%zmm6,%zmm25  // LO_2
+	vpxord	%zmm24,%zmm10,%zmm10  // sum(LO_{1,0})
+	vpclmulqdq	$0x00,%zmm30,%zmm7,%zmm26  // LO_3
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm10  // LO = sum(LO_{3,2,1,0})
+	vpclmulqdq	$0x01,%zmm27,%zmm4,%zmm24  // MI_0
+	vpclmulqdq	$0x01,%zmm28,%zmm5,%zmm25  // MI_1
+	vpclmulqdq	$0x01,%zmm29,%zmm6,%zmm26  // MI_2
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm30,%zmm7,%zmm25  // MI_3
+	vpclmulqdq	$0x10,%zmm27,%zmm4,%zmm26  // MI_4
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{4,3,2,1,0})
+	vpclmulqdq	$0x10,%zmm28,%zmm5,%zmm25  // MI_5
+	vpclmulqdq	$0x10,%zmm29,%zmm6,%zmm26  // MI_6
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{6,5,4,3,2,1,0})
+	vpclmulqdq	$0x01,%zmm10,%zmm31,%zmm26  // LO_L*(x^63 + x^62 + x^57)
+	vpclmulqdq	$0x10,%zmm30,%zmm7,%zmm25  // MI_7
+	vpxord	%zmm25,%zmm24,%zmm24  // MI = sum(MI_{7,6,5,4,3,2,1,0})
+	vpshufd	$0x4e,%zmm10,%zmm10  // Swap halves of LO
+	vpclmulqdq	$0x11,%zmm27,%zmm4,%zmm4  // HI_0
+	vpclmulqdq	$0x11,%zmm28,%zmm5,%zmm5  // HI_1
+	vpclmulqdq	$0x11,%zmm29,%zmm6,%zmm6  // HI_2
+	vpternlogd	$0x96,%zmm26,%zmm10,%zmm24  // Fold LO into MI
+	vpclmulqdq	$0x11,%zmm30,%zmm7,%zmm7  // HI_3
+	vpternlogd	$0x96,%zmm6,%zmm5,%zmm4  // sum(HI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm24,%zmm31,%zmm25  // MI_L*(x^63 + x^62 + x^57)
+	vpxord	%zmm7,%zmm4,%zmm10  // HI = sum(HI_{3,2,1,0})
+	vpshufd	$0x4e,%zmm24,%zmm24  // Swap halves of MI
+	vpternlogd	$0x96,%zmm25,%zmm24,%zmm10  // Fold MI into HI
+	vextracti32x4	$1,%zmm10,%xmm4
+	vextracti32x4	$2,%zmm10,%xmm5
+	vextracti32x4	$3,%zmm10,%xmm6
+	vpxord	%xmm4,%xmm10,%xmm10
+	vpternlogd	$0x96,%xmm5,%xmm6,%xmm10
+
+L$crypt_loop_4x_done__func1:
+// Check whether any data remains.
+	testq	%rdx,%rdx
+	jz	L$done__func1
+
+// The data length isn't a multiple of 256 bytes.  Process the remaining
+// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time.
+// Going one vector at a time may seem inefficient compared to having
+// separate code paths for each possible number of vectors remaining.
+// However, using a loop keeps the code size down, and it performs
+// surprising well; modern CPUs will start executing the next iteration
+// before the previous one finishes and also predict the number of loop
+// iterations.  For a similar reason, we roll up the AES rounds.
+// 
+// On the last iteration, the remaining length may be less than 64 bytes.
+// Handle this using masking.
+// 
+// Since there are enough key powers available for all remaining data,
+// there is no need to do a GHASH reduction after each iteration.
+// Instead, multiply each remaining block by its own key power, and only
+// do a GHASH reduction at the very end.
+
+// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N
+// is the number of blocks that remain.
+	movq	%rdx,%rax
+	negq	%rax
+	andq	$-16,%rax  // -round_up(DATALEN, 16)
+	leaq	256(%r9,%rax,1),%r8
+	vpxor	%xmm4,%xmm4,%xmm4
+	vpxor	%xmm5,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm6,%xmm6
+
+	cmpq	$64,%rdx
+	jb	L$partial_vec__func1
+
+L$crypt_loop_1x__func1:
+// Process a full 64-byte vector.
+
+// Encrypt a vector of counter blocks.
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpxord	%zmm13,%zmm0,%zmm0
+	leaq	16(%rcx),%rax
+L$vaesenc_loop_tail_full_vec__func1:
+	vbroadcasti32x4	(%rax),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	L$vaesenc_loop_tail_full_vec__func1
+	vaesenclast	%zmm14,%zmm0,%zmm0
+
+// XOR the data with the vector of keystream blocks.
+	vmovdqu8	(%rdi),%zmm1
+	vpxord	%zmm1,%zmm0,%zmm0
+	vmovdqu8	%zmm0,(%rsi)
+
+// Update GHASH with the ciphertext blocks, without reducing.
+	vmovdqu8	(%r8),%zmm30
+	vpshufb	%zmm8,%zmm0,%zmm0
+	vpxord	%zmm10,%zmm0,%zmm0
+	vpclmulqdq	$0x00,%zmm30,%zmm0,%zmm7  // a_L * b_L
+	vpclmulqdq	$0x01,%zmm30,%zmm0,%zmm1  // a_L * b_H
+	vpclmulqdq	$0x10,%zmm30,%zmm0,%zmm2  // a_H * b_L
+	vpclmulqdq	$0x11,%zmm30,%zmm0,%zmm3  // a_H * b_H
+	vpxord	%zmm7,%zmm4,%zmm4
+	vpternlogd	$0x96,%zmm2,%zmm1,%zmm5
+	vpxord	%zmm3,%zmm6,%zmm6
+
+	vpxor	%xmm10,%xmm10,%xmm10
+
+	addq	$64,%r8
+	addq	$64,%rdi
+	addq	$64,%rsi
+	subq	$64,%rdx
+	cmpq	$64,%rdx
+	jae	L$crypt_loop_1x__func1
+
+	testq	%rdx,%rdx
+	jz	L$reduce__func1
+
+L$partial_vec__func1:
+// Process a partial vector of length 1 <= DATALEN < 64.
+
+// Set the data mask %k1 to DATALEN 1's.
+// Set the key powers mask %k2 to round_up(DATALEN, 16) 1's.
+	movq	$-1,%rax
+	bzhiq	%rdx,%rax,%rax
+	kmovq	%rax,%k1
+	addq	$15,%rdx
+	andq	$-16,%rdx
+	movq	$-1,%rax
+	bzhiq	%rdx,%rax,%rax
+	kmovq	%rax,%k2
+
+// Encrypt one last vector of counter blocks.  This does not need to be
+// masked.  The counter does not need to be incremented here.
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpxord	%zmm13,%zmm0,%zmm0
+	leaq	16(%rcx),%rax
+L$vaesenc_loop_tail_partialvec__func1:
+	vbroadcasti32x4	(%rax),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	L$vaesenc_loop_tail_partialvec__func1
+	vaesenclast	%zmm14,%zmm0,%zmm0
+
+// XOR the data with the appropriate number of keystream bytes.
+	vmovdqu8	(%rdi),%zmm1{%k1}{z}
+	vpxord	%zmm1,%zmm0,%zmm0
+	vmovdqu8	%zmm0,(%rsi){%k1}
+
+// Update GHASH with the ciphertext block(s), without reducing.
+// 
+// In the case of DATALEN < 64, the ciphertext is zero-padded to 64
+// bytes.  (If decrypting, it's done by the above masked load.  If
+// encrypting, it's done by the below masked register-to-register move.)
+// Note that if DATALEN <= 48, there will be additional padding beyond
+// the padding of the last block specified by GHASH itself; i.e., there
+// may be whole block(s) that get processed by the GHASH multiplication
+// and reduction instructions but should not actually be included in the
+// GHASH.  However, any such blocks are all-zeroes, and the values that
+// they're multiplied with are also all-zeroes.  Therefore they just add
+// 0 * 0 = 0 to the final GHASH result, which makes no difference.
+	vmovdqu8	(%r8),%zmm30{%k2}{z}
+	vmovdqu8	%zmm0,%zmm1{%k1}{z}
+	vpshufb	%zmm8,%zmm1,%zmm0
+	vpxord	%zmm10,%zmm0,%zmm0
+	vpclmulqdq	$0x00,%zmm30,%zmm0,%zmm7  // a_L * b_L
+	vpclmulqdq	$0x01,%zmm30,%zmm0,%zmm1  // a_L * b_H
+	vpclmulqdq	$0x10,%zmm30,%zmm0,%zmm2  // a_H * b_L
+	vpclmulqdq	$0x11,%zmm30,%zmm0,%zmm3  // a_H * b_H
+	vpxord	%zmm7,%zmm4,%zmm4
+	vpternlogd	$0x96,%zmm2,%zmm1,%zmm5
+	vpxord	%zmm3,%zmm6,%zmm6
+
+
+L$reduce__func1:
+// Finally, do the GHASH reduction.
+	vpclmulqdq	$0x01,%zmm4,%zmm31,%zmm0
+	vpshufd	$0x4e,%zmm4,%zmm4
+	vpternlogd	$0x96,%zmm0,%zmm4,%zmm5
+	vpclmulqdq	$0x01,%zmm5,%zmm31,%zmm0
+	vpshufd	$0x4e,%zmm5,%zmm5
+	vpternlogd	$0x96,%zmm0,%zmm5,%zmm6
+
+	vextracti32x4	$1,%zmm6,%xmm0
+	vextracti32x4	$2,%zmm6,%xmm1
+	vextracti32x4	$3,%zmm6,%xmm2
+	vpxord	%xmm0,%xmm6,%xmm10
+	vpternlogd	$0x96,%xmm1,%xmm2,%xmm10
+
+
+L$done__func1:
+// Store the updated GHASH accumulator back to memory.
+	vpshufb	%xmm8,%xmm10,%xmm10
+	vmovdqu	%xmm10,(%r12)
+
+	vzeroupper  // This is needed after using ymm or zmm registers.
+	popq	%r12
+
+	ret
+
+
+
+.globl	_aes_gcm_dec_update_vaes_avx512
+.private_extern _aes_gcm_dec_update_vaes_avx512
+
+.p2align	5
+.alt_entry _aes_gcm_dec_update_vaes_avx512
+_aes_gcm_dec_update_vaes_avx512:
+
+
+_CET_ENDBR
+	pushq	%r12
+
+
+	movq	16(%rsp),%r12  // arg7
+// Load some constants.
+	vbroadcasti32x4	L$bswap_mask(%rip),%zmm8
+	vbroadcasti32x4	L$gfpoly(%rip),%zmm31
+
+// Load the GHASH accumulator and the starting counter.
+// BoringSSL passes these values in big endian format.
+	vmovdqu	(%r12),%xmm10
+	vpshufb	%xmm8,%xmm10,%xmm10
+	vbroadcasti32x4	(%r8),%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm12
+
+// Load the AES key length in bytes.  BoringSSL stores number of rounds
+// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20.
+	movl	240(%rcx),%r10d
+	leal	-20(,%r10,4),%r10d
+
+// Make RNDKEYLAST_PTR point to the last AES round key.  This is the
+// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
+// respectively.  Then load the zero-th and last round keys.
+	leaq	96(%rcx,%r10,4),%r11
+	vbroadcasti32x4	(%rcx),%zmm13
+	vbroadcasti32x4	(%r11),%zmm14
+
+// Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words.
+	vpaddd	L$ctr_pattern(%rip),%zmm12,%zmm12
+
+// Load 4 into all 128-bit lanes of LE_CTR_INC.
+	vbroadcasti32x4	L$inc_4blocks(%rip),%zmm11
+
+// If there are at least 256 bytes of data, then continue into the loop
+// that processes 256 bytes of data at a time.  Otherwise skip it.
+	cmpq	$256,%rdx
+	jb	L$crypt_loop_4x_done__func2
+
+// Load powers of the hash key.
+	vmovdqu8	256-256(%r9),%zmm27
+	vmovdqu8	256-192(%r9),%zmm28
+	vmovdqu8	256-128(%r9),%zmm29
+	vmovdqu8	256-64(%r9),%zmm30
+// Cache as many additional AES round keys as possible.
+	vbroadcasti32x4	-144(%r11),%zmm15
+	vbroadcasti32x4	-128(%r11),%zmm16
+	vbroadcasti32x4	-112(%r11),%zmm17
+	vbroadcasti32x4	-96(%r11),%zmm18
+	vbroadcasti32x4	-80(%r11),%zmm19
+	vbroadcasti32x4	-64(%r11),%zmm20
+	vbroadcasti32x4	-48(%r11),%zmm21
+	vbroadcasti32x4	-32(%r11),%zmm22
+	vbroadcasti32x4	-16(%r11),%zmm23
+
+L$crypt_loop_4x__func2:
+	vmovdqu8	0(%rdi),%zmm4
+	vmovdqu8	64(%rdi),%zmm5
+	vmovdqu8	128(%rdi),%zmm6
+	vmovdqu8	192(%rdi),%zmm7
+// Start the AES encryption of the counter blocks.
+// Increment le_ctr four times to generate four vectors of little-endian
+// counter blocks, swap each to big-endian, and store them in aesdata[0-3].
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm1
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm2
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpshufb	%zmm8,%zmm12,%zmm3
+	vpaddd	%zmm11,%zmm12,%zmm12
+
+// AES "round zero": XOR in the zero-th round key.
+	vpxord	%zmm13,%zmm0,%zmm0
+	vpxord	%zmm13,%zmm1,%zmm1
+	vpxord	%zmm13,%zmm2,%zmm2
+	vpxord	%zmm13,%zmm3,%zmm3
+
+	cmpl	$24,%r10d
+	jl	L$aes128__func2
+	je	L$aes192__func2
+// AES-256
+	vbroadcasti32x4	-208(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+	vbroadcasti32x4	-192(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+L$aes192__func2:
+	vbroadcasti32x4	-176(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+	vbroadcasti32x4	-160(%r11),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	vaesenc	%zmm9,%zmm1,%zmm1
+	vaesenc	%zmm9,%zmm2,%zmm2
+	vaesenc	%zmm9,%zmm3,%zmm3
+
+L$aes128__func2:
+
+// Prefetch the source data 512 bytes ahead into the L1 data cache, to
+// improve performance when the hardware prefetcher is disabled.  Assumes the
+// L1 data cache line size is 64 bytes (de facto standard on x86_64).
+	prefetcht0	512+0(%rdi)
+	prefetcht0	512+64(%rdi)
+	prefetcht0	512+128(%rdi)
+	prefetcht0	512+192(%rdi)
+
+// Finish the AES encryption of the counter blocks in AESDATA[0-3],
+// interleaved with the GHASH update of the ciphertext blocks in
+// GHASHDATA[0-3].
+	vpshufb	%zmm8,%zmm4,%zmm4
+	vpxord	%zmm10,%zmm4,%zmm4
+	vpshufb	%zmm8,%zmm5,%zmm5
+	vpshufb	%zmm8,%zmm6,%zmm6
+
+	vaesenc	%zmm15,%zmm0,%zmm0
+	vaesenc	%zmm15,%zmm1,%zmm1
+	vaesenc	%zmm15,%zmm2,%zmm2
+	vaesenc	%zmm15,%zmm3,%zmm3
+
+	vpshufb	%zmm8,%zmm7,%zmm7
+	vpclmulqdq	$0x00,%zmm27,%zmm4,%zmm10  // LO_0
+	vpclmulqdq	$0x00,%zmm28,%zmm5,%zmm24  // LO_1
+	vpclmulqdq	$0x00,%zmm29,%zmm6,%zmm25  // LO_2
+
+	vaesenc	%zmm16,%zmm0,%zmm0
+	vaesenc	%zmm16,%zmm1,%zmm1
+	vaesenc	%zmm16,%zmm2,%zmm2
+	vaesenc	%zmm16,%zmm3,%zmm3
+
+	vpxord	%zmm24,%zmm10,%zmm10  // sum(LO_{1,0})
+	vpclmulqdq	$0x00,%zmm30,%zmm7,%zmm26  // LO_3
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm10  // LO = sum(LO_{3,2,1,0})
+	vpclmulqdq	$0x01,%zmm27,%zmm4,%zmm24  // MI_0
+
+	vaesenc	%zmm17,%zmm0,%zmm0
+	vaesenc	%zmm17,%zmm1,%zmm1
+	vaesenc	%zmm17,%zmm2,%zmm2
+	vaesenc	%zmm17,%zmm3,%zmm3
+
+	vpclmulqdq	$0x01,%zmm28,%zmm5,%zmm25  // MI_1
+	vpclmulqdq	$0x01,%zmm29,%zmm6,%zmm26  // MI_2
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm30,%zmm7,%zmm25  // MI_3
+
+	vaesenc	%zmm18,%zmm0,%zmm0
+	vaesenc	%zmm18,%zmm1,%zmm1
+	vaesenc	%zmm18,%zmm2,%zmm2
+	vaesenc	%zmm18,%zmm3,%zmm3
+
+	vpclmulqdq	$0x10,%zmm27,%zmm4,%zmm26  // MI_4
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{4,3,2,1,0})
+	vpclmulqdq	$0x10,%zmm28,%zmm5,%zmm25  // MI_5
+	vpclmulqdq	$0x10,%zmm29,%zmm6,%zmm26  // MI_6
+
+	vaesenc	%zmm19,%zmm0,%zmm0
+	vaesenc	%zmm19,%zmm1,%zmm1
+	vaesenc	%zmm19,%zmm2,%zmm2
+	vaesenc	%zmm19,%zmm3,%zmm3
+
+	vpternlogd	$0x96,%zmm26,%zmm25,%zmm24  // sum(MI_{6,5,4,3,2,1,0})
+	vpclmulqdq	$0x01,%zmm10,%zmm31,%zmm26  // LO_L*(x^63 + x^62 + x^57)
+	vpclmulqdq	$0x10,%zmm30,%zmm7,%zmm25  // MI_7
+	vpxord	%zmm25,%zmm24,%zmm24  // MI = sum(MI_{7,6,5,4,3,2,1,0})
+
+	vaesenc	%zmm20,%zmm0,%zmm0
+	vaesenc	%zmm20,%zmm1,%zmm1
+	vaesenc	%zmm20,%zmm2,%zmm2
+	vaesenc	%zmm20,%zmm3,%zmm3
+
+	vpshufd	$0x4e,%zmm10,%zmm10  // Swap halves of LO
+	vpclmulqdq	$0x11,%zmm27,%zmm4,%zmm4  // HI_0
+	vpclmulqdq	$0x11,%zmm28,%zmm5,%zmm5  // HI_1
+	vpclmulqdq	$0x11,%zmm29,%zmm6,%zmm6  // HI_2
+
+	vaesenc	%zmm21,%zmm0,%zmm0
+	vaesenc	%zmm21,%zmm1,%zmm1
+	vaesenc	%zmm21,%zmm2,%zmm2
+	vaesenc	%zmm21,%zmm3,%zmm3
+
+	vpternlogd	$0x96,%zmm26,%zmm10,%zmm24  // Fold LO into MI
+	vpclmulqdq	$0x11,%zmm30,%zmm7,%zmm7  // HI_3
+	vpternlogd	$0x96,%zmm6,%zmm5,%zmm4  // sum(HI_{2,1,0})
+	vpclmulqdq	$0x01,%zmm24,%zmm31,%zmm25  // MI_L*(x^63 + x^62 + x^57)
+
+	vaesenc	%zmm22,%zmm0,%zmm0
+	vaesenc	%zmm22,%zmm1,%zmm1
+	vaesenc	%zmm22,%zmm2,%zmm2
+	vaesenc	%zmm22,%zmm3,%zmm3
+
+	vpxord	%zmm7,%zmm4,%zmm10  // HI = sum(HI_{3,2,1,0})
+	vpshufd	$0x4e,%zmm24,%zmm24  // Swap halves of MI
+	vpternlogd	$0x96,%zmm25,%zmm24,%zmm10  // Fold MI into HI
+
+	vaesenc	%zmm23,%zmm0,%zmm0
+	vaesenc	%zmm23,%zmm1,%zmm1
+	vaesenc	%zmm23,%zmm2,%zmm2
+	vaesenc	%zmm23,%zmm3,%zmm3
+
+
+	vextracti32x4	$1,%zmm10,%xmm4
+	vextracti32x4	$2,%zmm10,%xmm5
+	vextracti32x4	$3,%zmm10,%xmm6
+	vpxord	%xmm4,%xmm10,%xmm10
+	vpternlogd	$0x96,%xmm5,%xmm6,%xmm10
+
+	vpxord	0(%rdi),%zmm14,%zmm4
+	vpxord	64(%rdi),%zmm14,%zmm5
+	vpxord	128(%rdi),%zmm14,%zmm6
+	vpxord	192(%rdi),%zmm14,%zmm7
+	vaesenclast	%zmm4,%zmm0,%zmm4
+	vaesenclast	%zmm5,%zmm1,%zmm5
+	vaesenclast	%zmm6,%zmm2,%zmm6
+	vaesenclast	%zmm7,%zmm3,%zmm7
+	vmovdqu8	%zmm4,0(%rsi)
+	vmovdqu8	%zmm5,64(%rsi)
+	vmovdqu8	%zmm6,128(%rsi)
+	vmovdqu8	%zmm7,192(%rsi)
+
+	addq	$256,%rdi
+	addq	$256,%rsi
+	subq	$256,%rdx
+	cmpq	$256,%rdx
+	jae	L$crypt_loop_4x__func2
+L$crypt_loop_4x_done__func2:
+// Check whether any data remains.
+	testq	%rdx,%rdx
+	jz	L$done__func2
+
+// The data length isn't a multiple of 256 bytes.  Process the remaining
+// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time.
+// Going one vector at a time may seem inefficient compared to having
+// separate code paths for each possible number of vectors remaining.
+// However, using a loop keeps the code size down, and it performs
+// surprising well; modern CPUs will start executing the next iteration
+// before the previous one finishes and also predict the number of loop
+// iterations.  For a similar reason, we roll up the AES rounds.
+// 
+// On the last iteration, the remaining length may be less than 64 bytes.
+// Handle this using masking.
+// 
+// Since there are enough key powers available for all remaining data,
+// there is no need to do a GHASH reduction after each iteration.
+// Instead, multiply each remaining block by its own key power, and only
+// do a GHASH reduction at the very end.
+
+// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N
+// is the number of blocks that remain.
+	movq	%rdx,%rax
+	negq	%rax
+	andq	$-16,%rax  // -round_up(DATALEN, 16)
+	leaq	256(%r9,%rax,1),%r8
+	vpxor	%xmm4,%xmm4,%xmm4
+	vpxor	%xmm5,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm6,%xmm6
+
+	cmpq	$64,%rdx
+	jb	L$partial_vec__func2
+
+L$crypt_loop_1x__func2:
+// Process a full 64-byte vector.
+
+// Encrypt a vector of counter blocks.
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpaddd	%zmm11,%zmm12,%zmm12
+	vpxord	%zmm13,%zmm0,%zmm0
+	leaq	16(%rcx),%rax
+L$vaesenc_loop_tail_full_vec__func2:
+	vbroadcasti32x4	(%rax),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	L$vaesenc_loop_tail_full_vec__func2
+	vaesenclast	%zmm14,%zmm0,%zmm0
+
+// XOR the data with the vector of keystream blocks.
+	vmovdqu8	(%rdi),%zmm1
+	vpxord	%zmm1,%zmm0,%zmm0
+	vmovdqu8	%zmm0,(%rsi)
+
+// Update GHASH with the ciphertext blocks, without reducing.
+	vmovdqu8	(%r8),%zmm30
+	vpshufb	%zmm8,%zmm1,%zmm0
+	vpxord	%zmm10,%zmm0,%zmm0
+	vpclmulqdq	$0x00,%zmm30,%zmm0,%zmm7  // a_L * b_L
+	vpclmulqdq	$0x01,%zmm30,%zmm0,%zmm1  // a_L * b_H
+	vpclmulqdq	$0x10,%zmm30,%zmm0,%zmm2  // a_H * b_L
+	vpclmulqdq	$0x11,%zmm30,%zmm0,%zmm3  // a_H * b_H
+	vpxord	%zmm7,%zmm4,%zmm4
+	vpternlogd	$0x96,%zmm2,%zmm1,%zmm5
+	vpxord	%zmm3,%zmm6,%zmm6
+
+	vpxor	%xmm10,%xmm10,%xmm10
+
+	addq	$64,%r8
+	addq	$64,%rdi
+	addq	$64,%rsi
+	subq	$64,%rdx
+	cmpq	$64,%rdx
+	jae	L$crypt_loop_1x__func2
+
+	testq	%rdx,%rdx
+	jz	L$reduce__func2
+
+L$partial_vec__func2:
+// Process a partial vector of length 1 <= DATALEN < 64.
+
+// Set the data mask %k1 to DATALEN 1's.
+// Set the key powers mask %k2 to round_up(DATALEN, 16) 1's.
+	movq	$-1,%rax
+	bzhiq	%rdx,%rax,%rax
+	kmovq	%rax,%k1
+	addq	$15,%rdx
+	andq	$-16,%rdx
+	movq	$-1,%rax
+	bzhiq	%rdx,%rax,%rax
+	kmovq	%rax,%k2
+
+// Encrypt one last vector of counter blocks.  This does not need to be
+// masked.  The counter does not need to be incremented here.
+	vpshufb	%zmm8,%zmm12,%zmm0
+	vpxord	%zmm13,%zmm0,%zmm0
+	leaq	16(%rcx),%rax
+L$vaesenc_loop_tail_partialvec__func2:
+	vbroadcasti32x4	(%rax),%zmm9
+	vaesenc	%zmm9,%zmm0,%zmm0
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	L$vaesenc_loop_tail_partialvec__func2
+	vaesenclast	%zmm14,%zmm0,%zmm0
+
+// XOR the data with the appropriate number of keystream bytes.
+	vmovdqu8	(%rdi),%zmm1{%k1}{z}
+	vpxord	%zmm1,%zmm0,%zmm0
+	vmovdqu8	%zmm0,(%rsi){%k1}
+
+// Update GHASH with the ciphertext block(s), without reducing.
+// 
+// In the case of DATALEN < 64, the ciphertext is zero-padded to 64
+// bytes.  (If decrypting, it's done by the above masked load.  If
+// encrypting, it's done by the below masked register-to-register move.)
+// Note that if DATALEN <= 48, there will be additional padding beyond
+// the padding of the last block specified by GHASH itself; i.e., there
+// may be whole block(s) that get processed by the GHASH multiplication
+// and reduction instructions but should not actually be included in the
+// GHASH.  However, any such blocks are all-zeroes, and the values that
+// they're multiplied with are also all-zeroes.  Therefore they just add
+// 0 * 0 = 0 to the final GHASH result, which makes no difference.
+	vmovdqu8	(%r8),%zmm30{%k2}{z}
+
+	vpshufb	%zmm8,%zmm1,%zmm0
+	vpxord	%zmm10,%zmm0,%zmm0
+	vpclmulqdq	$0x00,%zmm30,%zmm0,%zmm7  // a_L * b_L
+	vpclmulqdq	$0x01,%zmm30,%zmm0,%zmm1  // a_L * b_H
+	vpclmulqdq	$0x10,%zmm30,%zmm0,%zmm2  // a_H * b_L
+	vpclmulqdq	$0x11,%zmm30,%zmm0,%zmm3  // a_H * b_H
+	vpxord	%zmm7,%zmm4,%zmm4
+	vpternlogd	$0x96,%zmm2,%zmm1,%zmm5
+	vpxord	%zmm3,%zmm6,%zmm6
+
+
+L$reduce__func2:
+// Finally, do the GHASH reduction.
+	vpclmulqdq	$0x01,%zmm4,%zmm31,%zmm0
+	vpshufd	$0x4e,%zmm4,%zmm4
+	vpternlogd	$0x96,%zmm0,%zmm4,%zmm5
+	vpclmulqdq	$0x01,%zmm5,%zmm31,%zmm0
+	vpshufd	$0x4e,%zmm5,%zmm5
+	vpternlogd	$0x96,%zmm0,%zmm5,%zmm6
+
+	vextracti32x4	$1,%zmm6,%xmm0
+	vextracti32x4	$2,%zmm6,%xmm1
+	vextracti32x4	$3,%zmm6,%xmm2
+	vpxord	%xmm0,%xmm6,%xmm10
+	vpternlogd	$0x96,%xmm1,%xmm2,%xmm10
+
+
+L$done__func2:
+// Store the updated GHASH accumulator back to memory.
+	vpshufb	%xmm8,%xmm10,%xmm10
+	vmovdqu	%xmm10,(%r12)
+
+	vzeroupper  // This is needed after using ymm or zmm registers.
+	popq	%r12
+
+	ret
+
+
+
+#endif