visaljs 0.0.2 → 0.0.4

package/runtime/third_party/boringssl/gen/bcm/ghash-x86_64-apple.S

@@ -0,0 +1,1134 @@
+// This file is generated from a similarly-named Perl script in the BoringSSL
+// source tree. Do not edit by hand.
+
+#include <openssl/asm_base.h>
+
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__)
+.text	
+.globl	_gcm_init_clmul
+.private_extern _gcm_init_clmul
+
+.p2align	4
+_gcm_init_clmul:
+
+
+_CET_ENDBR
+L$_init_clmul:
+	movdqu	(%rsi),%xmm2
+	pshufd	$78,%xmm2,%xmm2  // dword swap
+
+// <<1 twist
+	pshufd	$255,%xmm2,%xmm4  // broadcast uppermost dword
+	movdqa	%xmm2,%xmm3
+	psllq	$1,%xmm2
+	pxor	%xmm5,%xmm5  // 
+	psrlq	$63,%xmm3
+	pcmpgtd	%xmm4,%xmm5  // broadcast carry bit
+	pslldq	$8,%xmm3
+	por	%xmm3,%xmm2  // H<<=1
+
+// magic reduction
+	pand	L$0x1c2_polynomial(%rip),%xmm5
+	pxor	%xmm5,%xmm2  // if(carry) H^=0x1c2_polynomial
+
+// calculate H^2
+	pshufd	$78,%xmm2,%xmm6
+	movdqa	%xmm2,%xmm0
+	pxor	%xmm2,%xmm6
+	movdqa	%xmm0,%xmm1  // 
+	pshufd	$78,%xmm0,%xmm3
+	pxor	%xmm0,%xmm3  // 
+	pclmulqdq	$0x00,%xmm2,%xmm0  // ######
+	pclmulqdq	$0x11,%xmm2,%xmm1  // ######
+	pclmulqdq	$0x00,%xmm6,%xmm3  // ######
+	pxor	%xmm0,%xmm3  // 
+	pxor	%xmm1,%xmm3  // 
+
+	movdqa	%xmm3,%xmm4  // 
+	psrldq	$8,%xmm3
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm3,%xmm1
+	pxor	%xmm4,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	pshufd	$78,%xmm2,%xmm3
+	pshufd	$78,%xmm0,%xmm4
+	pxor	%xmm2,%xmm3  // Karatsuba pre-processing
+	movdqu	%xmm2,0(%rdi)  // save H
+	pxor	%xmm0,%xmm4  // Karatsuba pre-processing
+	movdqu	%xmm0,16(%rdi)  // save H^2
+	palignr	$8,%xmm3,%xmm4  // low part is H.lo^H.hi...
+	movdqu	%xmm4,32(%rdi)  // save Karatsuba "salt"
+	movdqa	%xmm0,%xmm1  // 
+	pshufd	$78,%xmm0,%xmm3
+	pxor	%xmm0,%xmm3  // 
+	pclmulqdq	$0x00,%xmm2,%xmm0  // ######
+	pclmulqdq	$0x11,%xmm2,%xmm1  // ######
+	pclmulqdq	$0x00,%xmm6,%xmm3  // ######
+	pxor	%xmm0,%xmm3  // 
+	pxor	%xmm1,%xmm3  // 
+
+	movdqa	%xmm3,%xmm4  // 
+	psrldq	$8,%xmm3
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm3,%xmm1
+	pxor	%xmm4,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm0,%xmm1  // 
+	pshufd	$78,%xmm0,%xmm3
+	pxor	%xmm0,%xmm3  // 
+	pclmulqdq	$0x00,%xmm2,%xmm0  // ######
+	pclmulqdq	$0x11,%xmm2,%xmm1  // ######
+	pclmulqdq	$0x00,%xmm6,%xmm3  // ######
+	pxor	%xmm0,%xmm3  // 
+	pxor	%xmm1,%xmm3  // 
+
+	movdqa	%xmm3,%xmm4  // 
+	psrldq	$8,%xmm3
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm3,%xmm1
+	pxor	%xmm4,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	pshufd	$78,%xmm5,%xmm3
+	pshufd	$78,%xmm0,%xmm4
+	pxor	%xmm5,%xmm3  // Karatsuba pre-processing
+	movdqu	%xmm5,48(%rdi)  // save H^3
+	pxor	%xmm0,%xmm4  // Karatsuba pre-processing
+	movdqu	%xmm0,64(%rdi)  // save H^4
+	palignr	$8,%xmm3,%xmm4  // low part is H^3.lo^H^3.hi...
+	movdqu	%xmm4,80(%rdi)  // save Karatsuba "salt"
+	ret
+
+
+
+.globl	_gcm_gmult_clmul
+.private_extern _gcm_gmult_clmul
+
+.p2align	4
+.alt_entry _gcm_gmult_clmul
+_gcm_gmult_clmul:
+
+_CET_ENDBR
+L$_gmult_clmul:
+	movdqu	(%rdi),%xmm0
+	movdqa	L$bswap_mask(%rip),%xmm5
+	movdqu	(%rsi),%xmm2
+	movdqu	32(%rsi),%xmm4
+	pshufb	%xmm5,%xmm0
+	movdqa	%xmm0,%xmm1  // 
+	pshufd	$78,%xmm0,%xmm3
+	pxor	%xmm0,%xmm3  // 
+	pclmulqdq	$0x00,%xmm2,%xmm0  // ######
+	pclmulqdq	$0x11,%xmm2,%xmm1  // ######
+	pclmulqdq	$0x00,%xmm4,%xmm3  // ######
+	pxor	%xmm0,%xmm3  // 
+	pxor	%xmm1,%xmm3  // 
+
+	movdqa	%xmm3,%xmm4  // 
+	psrldq	$8,%xmm3
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm3,%xmm1
+	pxor	%xmm4,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	pshufb	%xmm5,%xmm0
+	movdqu	%xmm0,(%rdi)
+	ret
+
+
+.globl	_gcm_ghash_clmul
+.private_extern _gcm_ghash_clmul
+
+.p2align	5
+.alt_entry _gcm_ghash_clmul
+_gcm_ghash_clmul:
+
+
+_CET_ENDBR
+L$_ghash_clmul:
+	movdqa	L$bswap_mask(%rip),%xmm10
+
+	movdqu	(%rdi),%xmm0
+	movdqu	(%rsi),%xmm2
+	movdqu	32(%rsi),%xmm7
+	pshufb	%xmm10,%xmm0
+
+	subq	$0x10,%rcx
+	jz	L$odd_tail
+
+	movdqu	16(%rsi),%xmm6
+	cmpq	$0x30,%rcx
+	jb	L$skip4x
+
+	subq	$0x30,%rcx
+	movq	$0xA040608020C0E000,%rax  // ((7..0)·0xE0)&0xff
+	movdqu	48(%rsi),%xmm14
+	movdqu	64(%rsi),%xmm15
+
+// ######
+// Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P
+// 
+	movdqu	48(%rdx),%xmm3
+	movdqu	32(%rdx),%xmm11
+	pshufb	%xmm10,%xmm3
+	pshufb	%xmm10,%xmm11
+	movdqa	%xmm3,%xmm5
+	pshufd	$78,%xmm3,%xmm4
+	pxor	%xmm3,%xmm4
+	pclmulqdq	$0x00,%xmm2,%xmm3
+	pclmulqdq	$0x11,%xmm2,%xmm5
+	pclmulqdq	$0x00,%xmm7,%xmm4
+
+	movdqa	%xmm11,%xmm13
+	pshufd	$78,%xmm11,%xmm12
+	pxor	%xmm11,%xmm12
+	pclmulqdq	$0x00,%xmm6,%xmm11
+	pclmulqdq	$0x11,%xmm6,%xmm13
+	pclmulqdq	$0x10,%xmm7,%xmm12
+	xorps	%xmm11,%xmm3
+	xorps	%xmm13,%xmm5
+	movups	80(%rsi),%xmm7
+	xorps	%xmm12,%xmm4
+
+	movdqu	16(%rdx),%xmm11
+	movdqu	0(%rdx),%xmm8
+	pshufb	%xmm10,%xmm11
+	pshufb	%xmm10,%xmm8
+	movdqa	%xmm11,%xmm13
+	pshufd	$78,%xmm11,%xmm12
+	pxor	%xmm8,%xmm0
+	pxor	%xmm11,%xmm12
+	pclmulqdq	$0x00,%xmm14,%xmm11
+	movdqa	%xmm0,%xmm1
+	pshufd	$78,%xmm0,%xmm8
+	pxor	%xmm0,%xmm8
+	pclmulqdq	$0x11,%xmm14,%xmm13
+	pclmulqdq	$0x00,%xmm7,%xmm12
+	xorps	%xmm11,%xmm3
+	xorps	%xmm13,%xmm5
+
+	leaq	64(%rdx),%rdx
+	subq	$0x40,%rcx
+	jc	L$tail4x
+
+	jmp	L$mod4_loop
+.p2align	5
+L$mod4_loop:
+	pclmulqdq	$0x00,%xmm15,%xmm0
+	xorps	%xmm12,%xmm4
+	movdqu	48(%rdx),%xmm11
+	pshufb	%xmm10,%xmm11
+	pclmulqdq	$0x11,%xmm15,%xmm1
+	xorps	%xmm3,%xmm0
+	movdqu	32(%rdx),%xmm3
+	movdqa	%xmm11,%xmm13
+	pclmulqdq	$0x10,%xmm7,%xmm8
+	pshufd	$78,%xmm11,%xmm12
+	xorps	%xmm5,%xmm1
+	pxor	%xmm11,%xmm12
+	pshufb	%xmm10,%xmm3
+	movups	32(%rsi),%xmm7
+	xorps	%xmm4,%xmm8
+	pclmulqdq	$0x00,%xmm2,%xmm11
+	pshufd	$78,%xmm3,%xmm4
+
+	pxor	%xmm0,%xmm8  // aggregated Karatsuba post-processing
+	movdqa	%xmm3,%xmm5
+	pxor	%xmm1,%xmm8  // 
+	pxor	%xmm3,%xmm4
+	movdqa	%xmm8,%xmm9  // 
+	pclmulqdq	$0x11,%xmm2,%xmm13
+	pslldq	$8,%xmm8
+	psrldq	$8,%xmm9  // 
+	pxor	%xmm8,%xmm0
+	movdqa	L$7_mask(%rip),%xmm8
+	pxor	%xmm9,%xmm1  // 
+	movq	%rax,%xmm9
+
+	pand	%xmm0,%xmm8  // 1st phase
+	pshufb	%xmm8,%xmm9  // 
+	pxor	%xmm0,%xmm9  // 
+	pclmulqdq	$0x00,%xmm7,%xmm12
+	psllq	$57,%xmm9  // 
+	movdqa	%xmm9,%xmm8  // 
+	pslldq	$8,%xmm9
+	pclmulqdq	$0x00,%xmm6,%xmm3
+	psrldq	$8,%xmm8  // 
+	pxor	%xmm9,%xmm0
+	pxor	%xmm8,%xmm1  // 
+	movdqu	0(%rdx),%xmm8
+
+	movdqa	%xmm0,%xmm9  // 2nd phase
+	psrlq	$1,%xmm0
+	pclmulqdq	$0x11,%xmm6,%xmm5
+	xorps	%xmm11,%xmm3
+	movdqu	16(%rdx),%xmm11
+	pshufb	%xmm10,%xmm11
+	pclmulqdq	$0x10,%xmm7,%xmm4
+	xorps	%xmm13,%xmm5
+	movups	80(%rsi),%xmm7
+	pshufb	%xmm10,%xmm8
+	pxor	%xmm9,%xmm1  // 
+	pxor	%xmm0,%xmm9
+	psrlq	$5,%xmm0
+
+	movdqa	%xmm11,%xmm13
+	pxor	%xmm12,%xmm4
+	pshufd	$78,%xmm11,%xmm12
+	pxor	%xmm9,%xmm0  // 
+	pxor	%xmm8,%xmm1
+	pxor	%xmm11,%xmm12
+	pclmulqdq	$0x00,%xmm14,%xmm11
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	movdqa	%xmm0,%xmm1
+	pclmulqdq	$0x11,%xmm14,%xmm13
+	xorps	%xmm11,%xmm3
+	pshufd	$78,%xmm0,%xmm8
+	pxor	%xmm0,%xmm8
+
+	pclmulqdq	$0x00,%xmm7,%xmm12
+	xorps	%xmm13,%xmm5
+
+	leaq	64(%rdx),%rdx
+	subq	$0x40,%rcx
+	jnc	L$mod4_loop
+
+L$tail4x:
+	pclmulqdq	$0x00,%xmm15,%xmm0
+	pclmulqdq	$0x11,%xmm15,%xmm1
+	pclmulqdq	$0x10,%xmm7,%xmm8
+	xorps	%xmm12,%xmm4
+	xorps	%xmm3,%xmm0
+	xorps	%xmm5,%xmm1
+	pxor	%xmm0,%xmm1  // aggregated Karatsuba post-processing
+	pxor	%xmm4,%xmm8
+
+	pxor	%xmm1,%xmm8  // 
+	pxor	%xmm0,%xmm1
+
+	movdqa	%xmm8,%xmm9  // 
+	psrldq	$8,%xmm8
+	pslldq	$8,%xmm9  // 
+	pxor	%xmm8,%xmm1
+	pxor	%xmm9,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	addq	$0x40,%rcx
+	jz	L$done
+	movdqu	32(%rsi),%xmm7
+	subq	$0x10,%rcx
+	jz	L$odd_tail
+L$skip4x:
+// ######
+// Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
+// [(H*Ii+1) + (H*Xi+1)] mod P =
+// [(H*Ii+1) + H^2*(Ii+Xi)] mod P
+// 
+	movdqu	(%rdx),%xmm8  // Ii
+	movdqu	16(%rdx),%xmm3  // Ii+1
+	pshufb	%xmm10,%xmm8
+	pshufb	%xmm10,%xmm3
+	pxor	%xmm8,%xmm0  // Ii+Xi
+
+	movdqa	%xmm3,%xmm5
+	pshufd	$78,%xmm3,%xmm4
+	pxor	%xmm3,%xmm4
+	pclmulqdq	$0x00,%xmm2,%xmm3
+	pclmulqdq	$0x11,%xmm2,%xmm5
+	pclmulqdq	$0x00,%xmm7,%xmm4
+
+	leaq	32(%rdx),%rdx  // i+=2
+	nop
+	subq	$0x20,%rcx
+	jbe	L$even_tail
+	nop
+	jmp	L$mod_loop
+
+.p2align	5
+L$mod_loop:
+	movdqa	%xmm0,%xmm1
+	movdqa	%xmm4,%xmm8
+	pshufd	$78,%xmm0,%xmm4  // 
+	pxor	%xmm0,%xmm4  // 
+
+	pclmulqdq	$0x00,%xmm6,%xmm0
+	pclmulqdq	$0x11,%xmm6,%xmm1
+	pclmulqdq	$0x10,%xmm7,%xmm4
+
+	pxor	%xmm3,%xmm0  // (H*Ii+1) + H^2*(Ii+Xi)
+	pxor	%xmm5,%xmm1
+	movdqu	(%rdx),%xmm9  // Ii
+	pxor	%xmm0,%xmm8  // aggregated Karatsuba post-processing
+	pshufb	%xmm10,%xmm9
+	movdqu	16(%rdx),%xmm3  // Ii+1
+
+	pxor	%xmm1,%xmm8
+	pxor	%xmm9,%xmm1  // "Ii+Xi", consume early
+	pxor	%xmm8,%xmm4
+	pshufb	%xmm10,%xmm3
+	movdqa	%xmm4,%xmm8  // 
+	psrldq	$8,%xmm8
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm8,%xmm1
+	pxor	%xmm4,%xmm0  // 
+
+	movdqa	%xmm3,%xmm5  // 
+
+	movdqa	%xmm0,%xmm9  // 1st phase
+	movdqa	%xmm0,%xmm8
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm8  // 
+	pclmulqdq	$0x00,%xmm2,%xmm3  // ######
+	psllq	$1,%xmm0
+	pxor	%xmm8,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm8  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm8  // 
+	pxor	%xmm9,%xmm0
+	pshufd	$78,%xmm5,%xmm4
+	pxor	%xmm8,%xmm1  // 
+	pxor	%xmm5,%xmm4  // 
+
+	movdqa	%xmm0,%xmm9  // 2nd phase
+	psrlq	$1,%xmm0
+	pclmulqdq	$0x11,%xmm2,%xmm5  // ######
+	pxor	%xmm9,%xmm1  // 
+	pxor	%xmm0,%xmm9
+	psrlq	$5,%xmm0
+	pxor	%xmm9,%xmm0  // 
+	leaq	32(%rdx),%rdx
+	psrlq	$1,%xmm0  // 
+	pclmulqdq	$0x00,%xmm7,%xmm4  // ######
+	pxor	%xmm1,%xmm0  // 
+
+	subq	$0x20,%rcx
+	ja	L$mod_loop
+
+L$even_tail:
+	movdqa	%xmm0,%xmm1
+	movdqa	%xmm4,%xmm8
+	pshufd	$78,%xmm0,%xmm4  // 
+	pxor	%xmm0,%xmm4  // 
+
+	pclmulqdq	$0x00,%xmm6,%xmm0
+	pclmulqdq	$0x11,%xmm6,%xmm1
+	pclmulqdq	$0x10,%xmm7,%xmm4
+
+	pxor	%xmm3,%xmm0  // (H*Ii+1) + H^2*(Ii+Xi)
+	pxor	%xmm5,%xmm1
+	pxor	%xmm0,%xmm8
+	pxor	%xmm1,%xmm8
+	pxor	%xmm8,%xmm4
+	movdqa	%xmm4,%xmm8  // 
+	psrldq	$8,%xmm8
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm8,%xmm1
+	pxor	%xmm4,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+	testq	%rcx,%rcx
+	jnz	L$done
+
+L$odd_tail:
+	movdqu	(%rdx),%xmm8  // Ii
+	pshufb	%xmm10,%xmm8
+	pxor	%xmm8,%xmm0  // Ii+Xi
+	movdqa	%xmm0,%xmm1  // 
+	pshufd	$78,%xmm0,%xmm3
+	pxor	%xmm0,%xmm3  // 
+	pclmulqdq	$0x00,%xmm2,%xmm0  // ######
+	pclmulqdq	$0x11,%xmm2,%xmm1  // ######
+	pclmulqdq	$0x00,%xmm7,%xmm3  // ######
+	pxor	%xmm0,%xmm3  // 
+	pxor	%xmm1,%xmm3  // 
+
+	movdqa	%xmm3,%xmm4  // 
+	psrldq	$8,%xmm3
+	pslldq	$8,%xmm4  // 
+	pxor	%xmm3,%xmm1
+	pxor	%xmm4,%xmm0  // 
+// 1st phase
+	movdqa	%xmm0,%xmm4  // 
+	movdqa	%xmm0,%xmm3
+	psllq	$5,%xmm0
+	pxor	%xmm0,%xmm3  // 
+	psllq	$1,%xmm0
+	pxor	%xmm3,%xmm0  // 
+	psllq	$57,%xmm0  // 
+	movdqa	%xmm0,%xmm3  // 
+	pslldq	$8,%xmm0
+	psrldq	$8,%xmm3  // 
+	pxor	%xmm4,%xmm0
+	pxor	%xmm3,%xmm1  // 
+
+// 2nd phase
+	movdqa	%xmm0,%xmm4
+	psrlq	$1,%xmm0
+	pxor	%xmm4,%xmm1  // 
+	pxor	%xmm0,%xmm4
+	psrlq	$5,%xmm0
+	pxor	%xmm4,%xmm0  // 
+	psrlq	$1,%xmm0  // 
+	pxor	%xmm1,%xmm0  // 
+L$done:
+	pshufb	%xmm10,%xmm0
+	movdqu	%xmm0,(%rdi)
+	ret
+
+
+
+.globl	_gcm_init_avx
+.private_extern _gcm_init_avx
+
+.p2align	5
+.alt_entry _gcm_init_avx
+_gcm_init_avx:
+
+
+_CET_ENDBR
+	vzeroupper
+
+	vmovdqu	(%rsi),%xmm2
+	vpshufd	$78,%xmm2,%xmm2  // dword swap
+
+// <<1 twist
+	vpshufd	$255,%xmm2,%xmm4  // broadcast uppermost dword
+	vpsrlq	$63,%xmm2,%xmm3
+	vpsllq	$1,%xmm2,%xmm2
+	vpxor	%xmm5,%xmm5,%xmm5  // 
+	vpcmpgtd	%xmm4,%xmm5,%xmm5  // broadcast carry bit
+	vpslldq	$8,%xmm3,%xmm3
+	vpor	%xmm3,%xmm2,%xmm2  // H<<=1
+
+// magic reduction
+	vpand	L$0x1c2_polynomial(%rip),%xmm5,%xmm5
+	vpxor	%xmm5,%xmm2,%xmm2  // if(carry) H^=0x1c2_polynomial
+
+	vpunpckhqdq	%xmm2,%xmm2,%xmm6
+	vmovdqa	%xmm2,%xmm0
+	vpxor	%xmm2,%xmm6,%xmm6
+	movq	$4,%r10  // up to H^8
+	jmp	L$init_start_avx
+.p2align	5
+L$init_loop_avx:
+	vpalignr	$8,%xmm3,%xmm4,%xmm5  // low part is H.lo^H.hi...
+	vmovdqu	%xmm5,-16(%rdi)  // save Karatsuba "salt"
+	vpunpckhqdq	%xmm0,%xmm0,%xmm3
+	vpxor	%xmm0,%xmm3,%xmm3  // 
+	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm1  // ######
+	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm0  // ######
+	vpclmulqdq	$0x00,%xmm6,%xmm3,%xmm3  // ######
+	vpxor	%xmm0,%xmm1,%xmm4  // 
+	vpxor	%xmm4,%xmm3,%xmm3  // 
+
+	vpslldq	$8,%xmm3,%xmm4  // 
+	vpsrldq	$8,%xmm3,%xmm3
+	vpxor	%xmm4,%xmm0,%xmm0  // 
+	vpxor	%xmm3,%xmm1,%xmm1
+	vpsllq	$57,%xmm0,%xmm3  // 1st phase
+	vpsllq	$62,%xmm0,%xmm4
+	vpxor	%xmm3,%xmm4,%xmm4  // 
+	vpsllq	$63,%xmm0,%xmm3
+	vpxor	%xmm3,%xmm4,%xmm4  // 
+	vpslldq	$8,%xmm4,%xmm3  // 
+	vpsrldq	$8,%xmm4,%xmm4
+	vpxor	%xmm3,%xmm0,%xmm0  // 
+	vpxor	%xmm4,%xmm1,%xmm1
+
+	vpsrlq	$1,%xmm0,%xmm4  // 2nd phase
+	vpxor	%xmm0,%xmm1,%xmm1
+	vpxor	%xmm4,%xmm0,%xmm0  // 
+	vpsrlq	$5,%xmm4,%xmm4
+	vpxor	%xmm4,%xmm0,%xmm0  // 
+	vpsrlq	$1,%xmm0,%xmm0  // 
+	vpxor	%xmm1,%xmm0,%xmm0  // 
+L$init_start_avx:
+	vmovdqa	%xmm0,%xmm5
+	vpunpckhqdq	%xmm0,%xmm0,%xmm3
+	vpxor	%xmm0,%xmm3,%xmm3  // 
+	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm1  // ######
+	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm0  // ######
+	vpclmulqdq	$0x00,%xmm6,%xmm3,%xmm3  // ######
+	vpxor	%xmm0,%xmm1,%xmm4  // 
+	vpxor	%xmm4,%xmm3,%xmm3  // 
+
+	vpslldq	$8,%xmm3,%xmm4  // 
+	vpsrldq	$8,%xmm3,%xmm3
+	vpxor	%xmm4,%xmm0,%xmm0  // 
+	vpxor	%xmm3,%xmm1,%xmm1
+	vpsllq	$57,%xmm0,%xmm3  // 1st phase
+	vpsllq	$62,%xmm0,%xmm4
+	vpxor	%xmm3,%xmm4,%xmm4  // 
+	vpsllq	$63,%xmm0,%xmm3
+	vpxor	%xmm3,%xmm4,%xmm4  // 
+	vpslldq	$8,%xmm4,%xmm3  // 
+	vpsrldq	$8,%xmm4,%xmm4
+	vpxor	%xmm3,%xmm0,%xmm0  // 
+	vpxor	%xmm4,%xmm1,%xmm1
+
+	vpsrlq	$1,%xmm0,%xmm4  // 2nd phase
+	vpxor	%xmm0,%xmm1,%xmm1
+	vpxor	%xmm4,%xmm0,%xmm0  // 
+	vpsrlq	$5,%xmm4,%xmm4
+	vpxor	%xmm4,%xmm0,%xmm0  // 
+	vpsrlq	$1,%xmm0,%xmm0  // 
+	vpxor	%xmm1,%xmm0,%xmm0  // 
+	vpshufd	$78,%xmm5,%xmm3
+	vpshufd	$78,%xmm0,%xmm4
+	vpxor	%xmm5,%xmm3,%xmm3  // Karatsuba pre-processing
+	vmovdqu	%xmm5,0(%rdi)  // save H^1,3,5,7
+	vpxor	%xmm0,%xmm4,%xmm4  // Karatsuba pre-processing
+	vmovdqu	%xmm0,16(%rdi)  // save H^2,4,6,8
+	leaq	48(%rdi),%rdi
+	subq	$1,%r10
+	jnz	L$init_loop_avx
+
+	vpalignr	$8,%xmm4,%xmm3,%xmm5  // last "salt" is flipped
+	vmovdqu	%xmm5,-16(%rdi)
+
+	vzeroupper
+	ret
+
+
+
+.globl	_gcm_gmult_avx
+.private_extern _gcm_gmult_avx
+
+.p2align	5
+.alt_entry _gcm_gmult_avx
+_gcm_gmult_avx:
+
+_CET_ENDBR
+	jmp	L$_gmult_clmul
+
+
+.globl	_gcm_ghash_avx
+.private_extern _gcm_ghash_avx
+
+.p2align	5
+.alt_entry _gcm_ghash_avx
+_gcm_ghash_avx:
+
+
+_CET_ENDBR
+	vzeroupper
+
+	vmovdqu	(%rdi),%xmm10  // load %xmm10
+	leaq	L$0x1c2_polynomial(%rip),%r10
+	leaq	64(%rsi),%rsi  // size optimization
+	vmovdqu	L$bswap_mask(%rip),%xmm13
+	vpshufb	%xmm13,%xmm10,%xmm10
+	cmpq	$0x80,%rcx
+	jb	L$short_avx
+	subq	$0x80,%rcx
+
+	vmovdqu	112(%rdx),%xmm14  // I[7]
+	vmovdqu	0-64(%rsi),%xmm6  // %xmm6^1
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vmovdqu	32-64(%rsi),%xmm7
+
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vmovdqu	96(%rdx),%xmm15  // I[6]
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpxor	%xmm14,%xmm9,%xmm9
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	16-64(%rsi),%xmm6  // %xmm6^2
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vmovdqu	80(%rdx),%xmm14  // I[5]
+	vpclmulqdq	$0x00,%xmm7,%xmm9,%xmm2
+	vpxor	%xmm15,%xmm8,%xmm8
+
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm3
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm4
+	vmovdqu	48-64(%rsi),%xmm6  // %xmm6^3
+	vpxor	%xmm14,%xmm9,%xmm9
+	vmovdqu	64(%rdx),%xmm15  // I[4]
+	vpclmulqdq	$0x10,%xmm7,%xmm8,%xmm5
+	vmovdqu	80-64(%rsi),%xmm7
+
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	64-64(%rsi),%xmm6  // %xmm6^4
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm9,%xmm2
+	vpxor	%xmm15,%xmm8,%xmm8
+
+	vmovdqu	48(%rdx),%xmm14  // I[3]
+	vpxor	%xmm3,%xmm0,%xmm0
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm3
+	vpxor	%xmm4,%xmm1,%xmm1
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm4
+	vmovdqu	96-64(%rsi),%xmm6  // %xmm6^5
+	vpxor	%xmm5,%xmm2,%xmm2
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpclmulqdq	$0x10,%xmm7,%xmm8,%xmm5
+	vmovdqu	128-64(%rsi),%xmm7
+	vpxor	%xmm14,%xmm9,%xmm9
+
+	vmovdqu	32(%rdx),%xmm15  // I[2]
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	112-64(%rsi),%xmm6  // %xmm6^6
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpclmulqdq	$0x00,%xmm7,%xmm9,%xmm2
+	vpxor	%xmm15,%xmm8,%xmm8
+
+	vmovdqu	16(%rdx),%xmm14  // I[1]
+	vpxor	%xmm3,%xmm0,%xmm0
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm3
+	vpxor	%xmm4,%xmm1,%xmm1
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm4
+	vmovdqu	144-64(%rsi),%xmm6  // %xmm6^7
+	vpxor	%xmm5,%xmm2,%xmm2
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpclmulqdq	$0x10,%xmm7,%xmm8,%xmm5
+	vmovdqu	176-64(%rsi),%xmm7
+	vpxor	%xmm14,%xmm9,%xmm9
+
+	vmovdqu	(%rdx),%xmm15  // I[0]
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	160-64(%rsi),%xmm6  // %xmm6^8
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x10,%xmm7,%xmm9,%xmm2
+
+	leaq	128(%rdx),%rdx
+	cmpq	$0x80,%rcx
+	jb	L$tail_avx
+
+	vpxor	%xmm10,%xmm15,%xmm15  // accumulate %xmm10
+	subq	$0x80,%rcx
+	jmp	L$oop8x_avx
+
+.p2align	5
+L$oop8x_avx:
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vmovdqu	112(%rdx),%xmm14  // I[7]
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpxor	%xmm15,%xmm8,%xmm8
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm10
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm11
+	vmovdqu	0-64(%rsi),%xmm6  // %xmm6^1
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm12
+	vmovdqu	32-64(%rsi),%xmm7
+	vpxor	%xmm14,%xmm9,%xmm9
+
+	vmovdqu	96(%rdx),%xmm15  // I[6]
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpxor	%xmm3,%xmm10,%xmm10  // collect result
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vxorps	%xmm4,%xmm11,%xmm11
+	vmovdqu	16-64(%rsi),%xmm6  // %xmm6^2
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpclmulqdq	$0x00,%xmm7,%xmm9,%xmm2
+	vpxor	%xmm5,%xmm12,%xmm12
+	vxorps	%xmm15,%xmm8,%xmm8
+
+	vmovdqu	80(%rdx),%xmm14  // I[5]
+	vpxor	%xmm10,%xmm12,%xmm12  // aggregated Karatsuba post-processing
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm3
+	vpxor	%xmm11,%xmm12,%xmm12
+	vpslldq	$8,%xmm12,%xmm9
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm4
+	vpsrldq	$8,%xmm12,%xmm12
+	vpxor	%xmm9,%xmm10,%xmm10
+	vmovdqu	48-64(%rsi),%xmm6  // %xmm6^3
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vxorps	%xmm12,%xmm11,%xmm11
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpclmulqdq	$0x10,%xmm7,%xmm8,%xmm5
+	vmovdqu	80-64(%rsi),%xmm7
+	vpxor	%xmm14,%xmm9,%xmm9
+	vpxor	%xmm2,%xmm5,%xmm5
+
+	vmovdqu	64(%rdx),%xmm15  // I[4]
+	vpalignr	$8,%xmm10,%xmm10,%xmm12  // 1st phase
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpxor	%xmm3,%xmm0,%xmm0
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	64-64(%rsi),%xmm6  // %xmm6^4
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm4,%xmm1,%xmm1
+	vpclmulqdq	$0x00,%xmm7,%xmm9,%xmm2
+	vxorps	%xmm15,%xmm8,%xmm8
+	vpxor	%xmm5,%xmm2,%xmm2
+
+	vmovdqu	48(%rdx),%xmm14  // I[3]
+	vpclmulqdq	$0x10,(%r10),%xmm10,%xmm10
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm3
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm4
+	vmovdqu	96-64(%rsi),%xmm6  // %xmm6^5
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x10,%xmm7,%xmm8,%xmm5
+	vmovdqu	128-64(%rsi),%xmm7
+	vpxor	%xmm14,%xmm9,%xmm9
+	vpxor	%xmm2,%xmm5,%xmm5
+
+	vmovdqu	32(%rdx),%xmm15  // I[2]
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpxor	%xmm3,%xmm0,%xmm0
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	112-64(%rsi),%xmm6  // %xmm6^6
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm4,%xmm1,%xmm1
+	vpclmulqdq	$0x00,%xmm7,%xmm9,%xmm2
+	vpxor	%xmm15,%xmm8,%xmm8
+	vpxor	%xmm5,%xmm2,%xmm2
+	vxorps	%xmm12,%xmm10,%xmm10
+
+	vmovdqu	16(%rdx),%xmm14  // I[1]
+	vpalignr	$8,%xmm10,%xmm10,%xmm12  // 2nd phase
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm3
+	vpshufb	%xmm13,%xmm14,%xmm14
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm4
+	vmovdqu	144-64(%rsi),%xmm6  // %xmm6^7
+	vpclmulqdq	$0x10,(%r10),%xmm10,%xmm10
+	vxorps	%xmm11,%xmm12,%xmm12
+	vpunpckhqdq	%xmm14,%xmm14,%xmm9
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x10,%xmm7,%xmm8,%xmm5
+	vmovdqu	176-64(%rsi),%xmm7
+	vpxor	%xmm14,%xmm9,%xmm9
+	vpxor	%xmm2,%xmm5,%xmm5
+
+	vmovdqu	(%rdx),%xmm15  // I[0]
+	vpclmulqdq	$0x00,%xmm6,%xmm14,%xmm0
+	vpshufb	%xmm13,%xmm15,%xmm15
+	vpclmulqdq	$0x11,%xmm6,%xmm14,%xmm1
+	vmovdqu	160-64(%rsi),%xmm6  // %xmm6^8
+	vpxor	%xmm12,%xmm15,%xmm15
+	vpclmulqdq	$0x10,%xmm7,%xmm9,%xmm2
+	vpxor	%xmm10,%xmm15,%xmm15  // accumulate %xmm10
+
+	leaq	128(%rdx),%rdx
+	subq	$0x80,%rcx
+	jnc	L$oop8x_avx
+
+	addq	$0x80,%rcx
+	jmp	L$tail_no_xor_avx
+
+.p2align	5
+L$short_avx:
+	vmovdqu	-16(%rdx,%rcx,1),%xmm14  // very last word
+	leaq	(%rdx,%rcx,1),%rdx
+	vmovdqu	0-64(%rsi),%xmm6  // %xmm6^1
+	vmovdqu	32-64(%rsi),%xmm7
+	vpshufb	%xmm13,%xmm14,%xmm15
+
+	vmovdqa	%xmm0,%xmm3  // subtle way to zero %xmm3,
+	vmovdqa	%xmm1,%xmm4  // %xmm4 and
+	vmovdqa	%xmm2,%xmm5  // %xmm5
+	subq	$0x10,%rcx
+	jz	L$tail_avx
+
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vmovdqu	-32(%rdx),%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vmovdqu	16-64(%rsi),%xmm6  // %xmm6^2
+	vpshufb	%xmm13,%xmm14,%xmm15
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+	vpsrldq	$8,%xmm7,%xmm7
+	subq	$0x10,%rcx
+	jz	L$tail_avx
+
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vmovdqu	-48(%rdx),%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vmovdqu	48-64(%rsi),%xmm6  // %xmm6^3
+	vpshufb	%xmm13,%xmm14,%xmm15
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+	vmovdqu	80-64(%rsi),%xmm7
+	subq	$0x10,%rcx
+	jz	L$tail_avx
+
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vmovdqu	-64(%rdx),%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vmovdqu	64-64(%rsi),%xmm6  // %xmm6^4
+	vpshufb	%xmm13,%xmm14,%xmm15
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+	vpsrldq	$8,%xmm7,%xmm7
+	subq	$0x10,%rcx
+	jz	L$tail_avx
+
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vmovdqu	-80(%rdx),%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vmovdqu	96-64(%rsi),%xmm6  // %xmm6^5
+	vpshufb	%xmm13,%xmm14,%xmm15
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+	vmovdqu	128-64(%rsi),%xmm7
+	subq	$0x10,%rcx
+	jz	L$tail_avx
+
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vmovdqu	-96(%rdx),%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vmovdqu	112-64(%rsi),%xmm6  // %xmm6^6
+	vpshufb	%xmm13,%xmm14,%xmm15
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+	vpsrldq	$8,%xmm7,%xmm7
+	subq	$0x10,%rcx
+	jz	L$tail_avx
+
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vmovdqu	-112(%rdx),%xmm14
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vmovdqu	144-64(%rsi),%xmm6  // %xmm6^7
+	vpshufb	%xmm13,%xmm14,%xmm15
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+	vmovq	184-64(%rsi),%xmm7
+	subq	$0x10,%rcx
+	jmp	L$tail_avx
+
+.p2align	5
+L$tail_avx:
+	vpxor	%xmm10,%xmm15,%xmm15  // accumulate %xmm10
+L$tail_no_xor_avx:
+	vpunpckhqdq	%xmm15,%xmm15,%xmm8
+	vpxor	%xmm0,%xmm3,%xmm3
+	vpclmulqdq	$0x00,%xmm6,%xmm15,%xmm0
+	vpxor	%xmm15,%xmm8,%xmm8
+	vpxor	%xmm1,%xmm4,%xmm4
+	vpclmulqdq	$0x11,%xmm6,%xmm15,%xmm1
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpclmulqdq	$0x00,%xmm7,%xmm8,%xmm2
+
+	vmovdqu	(%r10),%xmm12
+
+	vpxor	%xmm0,%xmm3,%xmm10
+	vpxor	%xmm1,%xmm4,%xmm11
+	vpxor	%xmm2,%xmm5,%xmm5
+
+	vpxor	%xmm10,%xmm5,%xmm5  // aggregated Karatsuba post-processing
+	vpxor	%xmm11,%xmm5,%xmm5
+	vpslldq	$8,%xmm5,%xmm9
+	vpsrldq	$8,%xmm5,%xmm5
+	vpxor	%xmm9,%xmm10,%xmm10
+	vpxor	%xmm5,%xmm11,%xmm11
+
+	vpclmulqdq	$0x10,%xmm12,%xmm10,%xmm9  // 1st phase
+	vpalignr	$8,%xmm10,%xmm10,%xmm10
+	vpxor	%xmm9,%xmm10,%xmm10
+
+	vpclmulqdq	$0x10,%xmm12,%xmm10,%xmm9  // 2nd phase
+	vpalignr	$8,%xmm10,%xmm10,%xmm10
+	vpxor	%xmm11,%xmm10,%xmm10
+	vpxor	%xmm9,%xmm10,%xmm10
+
+	cmpq	$0,%rcx
+	jne	L$short_avx
+
+	vpshufb	%xmm13,%xmm10,%xmm10
+	vmovdqu	%xmm10,(%rdi)
+	vzeroupper
+	ret
+
+
+
+.section	__DATA,__const
+
+.p2align	6
+ghash_constants:
+L$bswap_mask:
+.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
+L$0x1c2_polynomial:
+.byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
+L$7_mask:
+.long	7,0,7,0
+.p2align	6
+
+.byte	71,72,65,83,72,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
+.p2align	6
+.text	
+#endif