npm - visaljs - Versions diffs - 0.0.1 → 0.0.3 - Mend

visaljs 0.0.1 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1008) hide show

package/runtime/third_party/boringssl/gen/bcm/aesv8-gcm-armv8-linux.S ADDED Viewed

@@ -0,0 +1,2907 @@
+// This file is generated from a similarly-named Perl script in the BoringSSL
+// source tree. Do not edit by hand.
+#include <openssl/asm_base.h>
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__ELF__)
+#if __ARM_MAX_ARCH__ >= 8
+.arch	armv8.2-a+crypto+sha3
+.text
+.globl	aes_gcm_enc_kernel
+.hidden	aes_gcm_enc_kernel
+.type	aes_gcm_enc_kernel,%function
+.align	4
+aes_gcm_enc_kernel:
+	AARCH64_SIGN_LINK_REGISTER
+	stp	x29, x30, [sp, #-224]!
+	mov	x29, sp
+	ld1	{ v0.16b}, [x4]                                                 // .Load initial counter block
+	stp	x19, x20, [sp, #16]
+	mov	v1.16b, v0.16b                                               // Initialize ctr1-3 from ctr0
+	mov	v2.16b, v0.16b
+	mov	v3.16b, v0.16b
+	mov	x16, x4                                                       // Pointer to counter block in memory
+	mov	x8, x5                                                            // Pointer to AES key schedule context
+	stp	x21, x22, [sp, #32]
+    // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel
+	stp	d8, d9, [sp, #64]                                                  // Save Neon registers
+	stp	d10, d11, [sp, #80]
+	stp	d12, d13, [sp, #96]
+	stp	d14, d15, [sp, #112]
+	ldr	w17, [x8, #240]                                             // .Load number of AES rounds
+	add	x7, x8, x17, lsl #4                                     // Calculate pointer to the last round key
+	ldp	x13, x14, [x7]                                         // load round N key (for final XOR)
+	ldr	q31, [x7, #-16]                                          // load round N-1 key
+	add	x4, x0, x1, lsr #3                    // Calculate end of input
+	lsr	x5, x1, #3                               // Total byte length
+	mov	x15, x5
+	ldr	w12, [x16, #12]                                            // .Load counter's low 32 bits
+	sub	x5, x5, #1                       // byte_len - 1
+	ldr	q18, [x8, #0]                                                   // load rk0
+	and	x5, x5, #0xffffffffffffffc0      // Align main loop end to a multiple of 64 bytes
+	add	x5, x5, x0
+	rev	w12, w12                                                     // Reverse for big-endian increment
+	uxtw	x10, w12                                                       // Zero extend reversed w12 into x10 for final counter update
+    // Pre-compute this value instead of using two instructions to reconstruct it every iteration
+	mov	x21, #0xc200000000000000                                // GHASH reduction constant
+	str	x21, [sp, #128]
+    // We maintain four copies of ctr values on the stack. Each loop iteration we
+    // store the updated ctr value to the last four bytes (e.g., 160 + 12).
+    // We then load the four values. This avoids a singificant number of
+    // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we
+    // calculate and store the values one iteration ahead so they have time to
+    // drain before we load them.
+	str	q0,     [sp, #160]                                  // Store base counter for block 0-3
+	str	q0,     [sp, #176]
+	str	q0,     [sp, #192]
+	str	q0,     [sp, #208]
+    // Since we need the values right away don't go through the stack this first
+    // time. Manually insert the incremented big-endian counter values.
+	rev	w20, w12
+	mov	v0.s[3], w20                                             // ctr0 + 0
+	add	w20, w12, #1
+	rev	w20, w20
+	mov	v1.s[3], w20                                             // ctr0 + 1
+	add	w20, w12, #2
+	rev	w20, w20
+	mov	v2.s[3], w20                                             // ctr0 + 2
+	add	w20, w12, #3
+	rev	w20, w20
+	mov	v3.s[3], w20                                             // ctr0 + 3
+    // Calculate the ctr values for the *next* (not current) group of four
+    // blocks. Store the incremented parts to the stack.
+	add	w20, w12, #4
+	rev	w20, w20
+	str	w20, [sp, #172]                             // ctr0 + 4 for next iter
+	add	w20, w12, #5
+	rev	w20, w20
+	str	w20, [sp, #188]                             // ctr0 + 5 for next iter
+	add	w20, w12, #6
+	rev	w20, w20
+	str	w20, [sp, #204]                             // ctr0 + 6 for next iter
+	add	w20, w12, #7
+	rev	w20, w20
+	str	w20, [sp, #220]                             // ctr0 + 7 for next iter
+	add	w12, w12, #8                                                 // Advance counter past these two sets
+    // --- Start AES for first 4 blocks ---
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 0
+	ldp	q19, q20, [x8, #16]                                           // load rk1, rk2
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 0
+	ldp	q21, q22, [x8, #48]                                           // load rk3, rk4
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 0
+	ldp	q23, q24, [x8, #80]                                           // load rk5, rk6
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 0
+	ldp	q13, q14, [x6, #32]                                         // load H2, H3 (GHASH keys)
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 1
+	ldp	q25, q26, [x8, #112]                                          // load rk7, rk8
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 1
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 2
+	ext	v14.16b, v14.16b, v14.16b, #8                                      // Byte swap H3 for GHASH
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 2
+	ext	v13.16b, v13.16b, v13.16b, #8                                      // Byte swap H2 for GHASH
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 2
+	ldr	q15, [x6, #80]                                               // load H4
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 2
+	ext	v15.16b, v15.16b, v15.16b, #8                                      // Byte swap H4 for GHASH
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 3
+	ld1	{ v11.16b}, [x3]                                      // .Load initial GHASH accumulator (T)
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 3
+	ext	v11.16b, v11.16b, v11.16b, #8                             // Byte swap T for GHASH
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 3
+	rev64	v11.16b, v11.16b                                           // Correct byte order within 64-bit lanes
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 3
+	trn2	v17.2d,  v14.2d,    v15.2d                                      // Karatsuba key: H4_low | H3_low
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 4
+	ldr	q12, [x6]                                                    // load H1
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 4
+	ext	v12.16b, v12.16b, v12.16b, #8                                      // Byte swap H1 for GHASH
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 4
+	trn1	v9.2d, v14.2d,    v15.2d                                      // Karatsuba key: H4_high | H3_high
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 4
+	trn2	v16.2d,  v12.2d,    v13.2d                                      // Karatsuba key: H2_low | H1_low
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 5
+	ldr	q30, [x7]                                                // Preload round N key for final EOR
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 5
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 5
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 5
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 6
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 6
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 6
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 6
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 7
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 7
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 7
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 7
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 8
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 8
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 8
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	b.lt	.Lenc_finish_first_blocks                                         // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	b.eq	.Lenc_finish_first_blocks                                         // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+.Lenc_finish_first_blocks:
+	cmp	x0, x5                                    // check if we have <= 4 blocks to process in the tail
+	eor	v17.16b, v17.16b, v9.16b                                   // Karatsuba key: H3^H4
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	trn1	v8.2d,  v12.2d, v13.2d                                 // Karatsuba key: H2_high | H1_high
+	eor	v16.16b, v16.16b, v8.16b                            // Karatsuba key: H1^H2
+	b.ge	.Lenc_tail                                                        // handle tail if no more full 4-block sets
+	ldp	q6, q7, [x0, #32]                                  // AES blocks 2,3 load plaintext
+	ldp	q4, q5, [x0], #64                                  // AES blocks 0,1 load plaintext
+    // Compute and store first 4 ciphertext blocks
+	eor	v4.16b, v4.16b, v30.16b
+	eor	v4.16b, v4.16b, v0.16b                                // AES block 0 - result = PT ^ AES(ctr0)
+	eor	v5.16b, v5.16b, v30.16b
+	eor	v5.16b, v5.16b, v1.16b                                // AES block 1 - result = PT ^ AES(ctr1)
+	eor	v6.16b, v6.16b, v30.16b
+	eor	v6.16b, v6.16b, v2.16b                                // AES block 2 - result = PT ^ AES(ctr2)
+	eor	v7.16b, v7.16b, v30.16b
+	eor	v7.16b, v7.16b, v3.16b                                // AES block 3 - result = PT ^ AES(ctr3)
+	st1	{ v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64  // AES blocks 0-3 - store result
+    // Load counter values for the second iteration from the stack
+	ldp	q0, q1, [sp, #160]
+	ldp	q2, q3, [sp, #192]
+    // Prepare and store counter values for the third iteration
+	rev	w20, w12
+	str	w20, [sp, #172]                             // ctr + 8
+	add	w20, w12, #1
+	rev	w20, w20
+	str	w20, [sp, #188]                             // ctr + 9
+	add	w20, w12, #2
+	rev	w20, w20
+	str	w20, [sp, #204]                             // ctr + 10
+	add	w20, w12, #3
+	rev	w20, w20
+	str	w20, [sp, #220]                             // ctr + 11
+	add	w12, w12, #4                                                 // Advance counter base
+	cmp	x0, x5                                    // check if we have <= 4 blocks remaining
+	b.ge	.Lenc_prepretail                                                  // go to prepretail if < 2 full loops left
+.Lenc_main_loop:	//	main loop start (processes 4 blocks per iteration)
+    // --- AES Pipeline for blocks 4k+4 to 4k+7 ---
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 0
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 0
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 0
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 0
+	ldr	d8, [sp, #128]                      // .Load GHASH reduction constant
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 1
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 1
+    // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 ---
+	rev64	v4.16b, v4.16b                                             // GHASH block 4k - Byte swap CT
+	rev64	v5.16b, v5.16b                                             // GHASH block 4k+1 - Byte swap CT
+	rev64	v6.16b, v6.16b                                             // GHASH block 4k+2 - Byte swap CT
+	rev64	v7.16b, v7.16b                                             // GHASH block 4k+3 - Byte swap CT
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 2
+	ext	v11.16b, v11.16b, v11.16b, #8                             // GHASH - prepare acc for XOR
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 2
+	eor	v4.16b, v4.16b, v11.16b                                   // GHASH block 4k - Y_i = CT_i ^ Y_{i-1}
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 2
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 4k - low
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 2
+	pmull2	v9.1q, v6.2d, v13.2d                                     // GHASH block 4k+2 - high
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 3
+	mov	d10, v17.d[1]                                                // GHASH block 4k - mid Karatsuba key
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 3
+	mov	d20, v4.d[1]                                             // GHASH block 4k - mid
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 3
+	eor	v20.8b, v20.8b, v4.8b                               // GHASH block 4k - mid
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 3
+	mov	d21, v5.d[1]                                             // GHASH block 4k+1 - mid
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 4
+	eor	v21.8b, v21.8b, v5.8b                               // GHASH block 4k+1 - mid
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 4
+	pmull	v10.1q, v20.1d, v10.1d                               // GHASH block 4k - mid
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 4
+	pmull	v21.1q, v21.1d, v17.1d                             // GHASH block 4k+1 - mid
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 4
+	eor	v10.16b, v10.16b, v21.16b                              // GHASH block 4k+1 - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 5
+	ext	v22.16b, v22.16b, v6.16b, #8                        // GHASH block 4k+2 - mid
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 5
+	eor	v22.16b, v22.16b, v6.16b                            // GHASH block 4k+2 - mid
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 5
+	pmull2	v22.1q, v22.2d, v16.2d                            // GHASH block 4k+2 - mid
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 5
+	mov	d23, v7.d[1]                                             // GHASH block 4k+3 - mid
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 6
+	eor	v23.8b, v23.8b, v7.8b                               // GHASH block 4k+3 - mid
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 6
+	pmull	v23.1q, v23.1d, v16.1d                             // GHASH block 4k+3 - mid
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 6
+	eor	v10.16b, v10.16b, v22.16b
+	eor	v10.16b, v10.16b, v23.16b                       // GHASH block 4k+2/3 - mid
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 6
+	pmull2	v22.1q, v4.2d, v15.2d                                  // GHASH block 4k - high
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 7
+	pmull2	v21.1q, v7.2d, v12.2d                                  // GHASH block 4k+3 - high
+	eor	v22.16b, v22.16b, v21.16b                        // GHASH block 4k+3 - high
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 7
+	pmull2	v23.1q, v5.2d, v14.2d                                  // GHASH block 4k+1 - high
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 7
+	pmull	v21.1q, v6.1d, v13.1d                                   // GHASH block 4k+2 - low
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 7
+	pmull	v20.1q, v5.1d, v14.1d                                   // GHASH block 4k+1 - low
+	eor	v9.16b, v9.16b, v22.16b
+	eor	v9.16b, v9.16b, v23.16b                       // GHASH block 4k/1/2/3 - high
+	pmull	v22.1q, v7.1d, v12.1d                                   // GHASH block 4k+3 - low
+	ldp	q6, q7, [x0, #32]
+	ldp	q4, q5, [x0], #64
+	eor	v20.16b, v20.16b, v21.16b                        // GHASH block 4k+1 - low
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 8
+	eor	v11.16b, v11.16b, v22.16b
+	eor	v11.16b, v11.16b, v20.16b                       // GHASH block 4k/1/2/3 - low
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 8
+	eor	v10.16b, v10.16b, v9.16b
+	eor	v10.16b, v10.16b, v11.16b                             // MODULO - karatsuba tidy up
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 8
+	pmull	v20.1q, v9.1d, v8.1d                        // MODULO - top 64b align with mid
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	b.lt	.Lenc_main_loop_continue                                          // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 10
+	b.eq	.Lenc_main_loop_continue                                          // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 12
+.Lenc_main_loop_continue:
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor	v10.16b, v10.16b, v20.16b
+	eor	v10.16b, v10.16b, v9.16b                          // MODULO - fold into mid
+	pmull	v9.1q, v10.1d, v8.1d                           // MODULO - mid 64b align with low
+	ext	v20.16b, v10.16b, v10.16b, #8                          // MODULO - other mid alignment
+	eor	v11.16b, v9.16b, v11.16b
+	eor	v11.16b, v11.16b, v20.16b                          // MODULO - fold into low
+	aese	v0.16b, v31.16b                                             // AES block 4k+4 - round N-1
+	eor	v4.16b, v4.16b, v30.16b
+	eor	v4.16b, v4.16b, v0.16b                                // AES block 4k+4 - result
+	aese	v1.16b, v31.16b                                             // AES block 4k+5 - round N-1
+	eor	v5.16b, v5.16b, v30.16b
+	eor	v5.16b, v5.16b, v1.16b                                // AES block 4k+5 - result
+	aese	v2.16b, v31.16b                                             // AES block 4k+6 - round N-1
+	eor	v6.16b, v6.16b, v30.16b
+	eor	v6.16b, v6.16b, v2.16b                                // AES block 4k+6 - result
+	aese	v3.16b, v31.16b                                             // AES block 4k+7 - round N-1
+	eor	v7.16b, v7.16b, v30.16b
+	eor	v7.16b, v7.16b, v3.16b                                // AES block 4k+7 - result
+	ldp	q0, q1, [sp, #160]
+	ldp	q2, q3, [sp, #192]
+    // We used these registers as temporaries above so reload the RKs.
+	ldp	q20, q21, [x8, #32]                                           // load rk2, rk3
+	ldp	q22, q23, [x8, #64]                                           // load rk4, rk5
+	st1	{ v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64  // AES blocks 4k+4-7 - store result
+	rev	w20, w12
+	str	w20, [sp, #172]
+	add	w20, w12, #1
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w12, #2
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w12, #3
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w12, w12, #4
+	cmp	x0, x5                                    // .LOOP CONTROL
+	b.lt	.Lenc_main_loop
+.Lenc_prepretail:	//	PREPRETAIL
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 0
+	rev64	v6.16b, v6.16b                                             // GHASH block 2
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 0
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 0
+	rev64	v4.16b, v4.16b                                             // GHASH block 0
+	ext	v11.16b, v11.16b, v11.16b, #8                             // PRE 0
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 1
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 1
+	eor	v4.16b, v4.16b, v11.16b                                   // PRE 1
+	rev64	v5.16b, v5.16b                                             // GHASH block 1
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 2
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 0
+	mov	d10, v17.d[1]                                                // GHASH block 0 - mid Karatsuba key
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 1
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 0 - low
+	mov	d8, v4.d[1]                                         // GHASH block 0 - mid
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH block 0 - high
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 3
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 2
+	eor	v8.8b, v8.8b, v4.8b                       // GHASH block 0 - mid
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 2
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 1
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 3
+	pmull	v10.1q, v8.1d, v10.1d                           // GHASH block 0 - mid
+	pmull2	v4.1q, v5.2d, v14.2d                                      // GHASH block 1 - high
+	pmull	v8.1q, v5.1d, v14.1d                               // GHASH block 1 - low
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 2
+	eor	v9.16b, v9.16b, v4.16b                                  // GHASH block 1 - high
+	mov	d4, v5.d[1]                                                 // GHASH block 1 - mid
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 3
+	eor	v11.16b, v11.16b, v8.16b                          // GHASH block 1 - low
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 3
+	eor	v4.8b, v4.8b, v5.8b                                       // GHASH block 1 - mid
+	mov	d8, v6.d[1]                                         // GHASH block 2 - mid
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 4
+	rev64	v7.16b, v7.16b                                             // GHASH block 3
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 4
+	pmull	v4.1q, v4.1d, v17.1d                                     // GHASH block 1 - mid
+	eor	v8.8b, v8.8b, v6.8b                       // GHASH block 2 - mid
+	pmull	v5.1q, v6.1d, v13.1d                                       // GHASH block 2 - low
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 5
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 4
+	eor	v10.16b, v10.16b, v4.16b                                  // GHASH block 1 - mid
+	pmull2	v4.1q, v6.2d, v13.2d                                      // GHASH block 2 - high
+	eor	v11.16b, v11.16b, v5.16b                                  // GHASH block 2 - low
+	ins	v8.d[1], v8.d[0]                             // GHASH block 2 - mid
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 5
+	eor	v9.16b, v9.16b, v4.16b                                  // GHASH block 2 - high
+	mov	d4, v7.d[1]                                                 // GHASH block 3 - mid
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 4
+	pmull2	v8.1q, v8.2d, v16.2d                    // GHASH block 2 - mid
+	eor	v4.8b, v4.8b, v7.8b                                       // GHASH block 3 - mid
+	pmull2	v5.1q, v7.2d, v12.2d                                      // GHASH block 3 - high
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 5
+	pmull	v4.1q, v4.1d, v16.1d                                     // GHASH block 3 - mid
+	eor	v10.16b, v10.16b, v8.16b                          // GHASH block 2 - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 5
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 6
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 6
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 6
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 6
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 7
+	eor	v9.16b, v9.16b, v5.16b                                  // GHASH block 3 - high
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 7
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 7
+	ldr	d8, [sp, #128]
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 8
+	eor	v10.16b, v10.16b, v4.16b                                  // GHASH block 3 - mid
+	pmull	v6.1q, v7.1d, v12.1d                                       // GHASH block 3 - low
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 8
+	eor	v11.16b, v11.16b, v6.16b                                  // GHASH block 3 - low
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 7
+	eor	v10.16b, v10.16b, v9.16b                                 // karatsuba tidy up
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 8
+	pmull	v4.1q, v9.1d, v8.1d
+	ext	v9.16b, v9.16b, v9.16b, #8
+	eor	v10.16b, v10.16b, v11.16b
+	b.lt	.Lenc_finish_prepretail                                           // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	b.eq	.Lenc_finish_prepretail                                           // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+.Lenc_finish_prepretail:
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	eor	v10.16b, v10.16b, v4.16b
+	eor	v10.16b, v10.16b, v9.16b
+	pmull	v4.1q, v10.1d, v8.1d
+	ext	v10.16b, v10.16b, v10.16b, #8
+	eor	v11.16b, v11.16b, v4.16b
+	eor	v11.16b, v11.16b, v10.16b
+.Lenc_tail:	//	TAIL: Process remaining 0 to 3 blocks
+	ext	v8.16b, v11.16b, v11.16b, #8                      // Save current GHASH state for partial tag feed-in
+	sub	x5, x4, x0                    // main_end_input_ptr is number of bytes left to process
+	ldp	x6, x7, [x0], #16                            // AES block 0 - load plaintext
+	eor	x6, x6, x13                                       // AES block 0 - round N low
+	eor	x7, x7, x14                                       // AES block 0 - round N high
+	cmp	x5, #48
+	fmov	d4, x6                                                 // AES block 0 - mov low
+	fmov	v4.d[1], x7                                             // AES block 0 - mov high
+	eor	v5.16b, v4.16b, v0.16b                                    // AES block 0 - result
+	b.gt	.Lenc_blocks_more_than_3
+	cmp	x5, #32
+	mov	v3.16b, v2.16b
+	movi	v11.8b, #0
+	movi	v9.8b, #0
+	mov	v2.16b, v1.16b
+	movi	v10.8b, #0
+	b.gt	.Lenc_blocks_more_than_2
+	mov	v3.16b, v1.16b
+	cmp	x5, #16
+	b.gt	.Lenc_blocks_more_than_1
+	b	.Lenc_blocks_less_than_1
+.Lenc_blocks_more_than_3:	//	blocks left >  3
+	st1	{ v5.16b}, [x2], #16                                   // AES final-2 block - store result
+	ldp	x6, x7, [x0], #16                            // AES final-2 block - load input low & high
+	rev64	v4.16b, v5.16b                                             // GHASH final-3 block
+	eor	x6, x6, x13                                       // AES final-2 block - round N low
+	eor	v4.16b, v4.16b, v8.16b                            // feed in partial tag
+	eor	x7, x7, x14                                       // AES final-2 block - round N high
+	mov	d22, v4.d[1]                                             // GHASH final-3 block - mid
+	fmov	d5, x6                                                 // AES final-2 block - mov low
+	fmov	v5.d[1], x7                                             // AES final-2 block - mov high
+	eor	v22.8b, v22.8b, v4.8b                               // GHASH final-3 block - mid
+	movi	v8.8b, #0                                              // suppress further partial tag feed in
+	mov	d10, v17.d[1]                                                // GHASH final-3 block - mid
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH final-3 block - low
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH final-3 block - high
+	pmull	v10.1q, v22.1d, v10.1d                               // GHASH final-3 block - mid
+	eor	v5.16b, v5.16b, v1.16b                                    // AES final-2 block - result
+.Lenc_blocks_more_than_2:	//	blocks left >  2
+	st1	{ v5.16b}, [x2], #16                                   // AES final-2 block - store result
+	ldp	x6, x7, [x0], #16                            // AES final-1 block - load input low & high
+	rev64	v4.16b, v5.16b                                             // GHASH final-2 block
+	eor	x6, x6, x13                                       // AES final-1 block - round N low
+	eor	v4.16b, v4.16b, v8.16b                            // feed in partial tag
+	fmov	d5, x6                                                 // AES final-1 block - mov low
+	eor	x7, x7, x14                                       // AES final-1 block - round N high
+	fmov	v5.d[1], x7                                             // AES final-1 block - mov high
+	movi	v8.8b, #0                                              // suppress further partial tag feed in
+	pmull2	v20.1q, v4.2d, v14.2d                                  // GHASH final-2 block - high
+	mov	d22, v4.d[1]                                             // GHASH final-2 block - mid
+	pmull	v21.1q, v4.1d, v14.1d                                   // GHASH final-2 block - low
+	eor	v22.8b, v22.8b, v4.8b                               // GHASH final-2 block - mid
+	eor	v5.16b, v5.16b, v2.16b                                    // AES final-1 block - result
+	eor	v9.16b, v9.16b, v20.16b                              // GHASH final-2 block - high
+	pmull	v22.1q, v22.1d, v17.1d                             // GHASH final-2 block - mid
+	eor	v11.16b, v11.16b, v21.16b                              // GHASH final-2 block - low
+	eor	v10.16b, v10.16b, v22.16b                              // GHASH final-2 block - mid
+.Lenc_blocks_more_than_1:	//	blocks left >  1
+	st1	{ v5.16b}, [x2], #16                                   // AES final-1 block - store result
+	rev64	v4.16b, v5.16b                                             // GHASH final-1 block: Byte Swap CT
+	ldp	x6, x7, [x0], #16                            // AES final block - load plaintext
+	eor	v4.16b, v4.16b, v8.16b                            // Feed in partial tag
+	movi	v8.8b, #0                                              // Clear for next block
+	eor	x6, x6, x13                                       // AES final block - round N low
+	mov	d22, v4.d[1]                                             // GHASH final-1 block - mid
+	pmull2	v20.1q, v4.2d, v13.2d                                  // GHASH final-1 block - high
+	eor	x7, x7, x14                                       // AES final block - round N high
+	eor	v22.8b, v22.8b, v4.8b                               // GHASH final-1 block - mid
+	eor	v9.16b, v9.16b, v20.16b                              // GHASH final-1 block - high
+	ins	v22.d[1], v22.d[0]                                     // GHASH final-1 block - mid
+	fmov	d5, x6                                                 // AES final block - mov low
+	fmov	v5.d[1], x7                                             // AES final block - mov high
+	pmull2	v22.1q, v22.2d, v16.2d                            // GHASH final-1 block - mid
+	pmull	v21.1q, v4.1d, v13.1d                                   // GHASH final-1 block - low
+	eor	v5.16b, v5.16b, v3.16b                                    // AES final block - result
+	eor	v10.16b, v10.16b, v22.16b                              // GHASH final-1 block - mid
+	eor	v11.16b, v11.16b, v21.16b                              // GHASH final-1 block - low
+.Lenc_blocks_less_than_1:	//	.Last partial block handling
+	add	x10, x10, x1, lsr #7                                    // Calculate the updated counter based on the number of 16B chunks we processed
+	rev	w10, w10
+	str	w10, [x16, #12]                                              // store the updated counter
+	and	x1, x1, #127                                     // bit_length %= 128
+	mvn	x13, xzr                                                        // Mask for low 64 bits
+	sub	x1, x1, #128                                     //
+	neg	x1, x1                                           // Valid bits in the last block (1-128)
+	ldr	q18, [x2]                                  // .Load destination for merging
+	mvn	x14, xzr                                                        // Mask for high 64 bits
+	and	x1, x1, #127                                     // bit_length %= 128
+	lsr	x14, x14, x1                                        // rkN_h is mask for top 64b of last block
+	cmp	x1, #64
+	csel	x6, x13, x14, lt
+	csel	x7, x14, xzr, lt
+	fmov	d0, x6                                                 // ctr0d is mask for last block
+	fmov	v0.d[1], x7
+	and	v5.16b, v5.16b, v0.16b                                    // Mask out unused bits of the last CT block
+	rev64	v4.16b, v5.16b                                             // GHASH final block - byte swap
+	eor	v4.16b, v4.16b, v8.16b                            // Feed in partial tag
+	bif	v5.16b, v18.16b, v0.16b                        // Bitwise Insert: merge with existing data at output_ptr
+	pmull2	v20.1q, v4.2d, v12.2d                                  // GHASH final block - high
+	mov	d8, v4.d[1]                                         // GHASH final block - mid
+	pmull	v21.1q, v4.1d, v12.1d                                   // GHASH final block - low
+	eor	v9.16b, v9.16b, v20.16b                              // GHASH final block - high
+	eor	v8.8b, v8.8b, v4.8b                       // GHASH final block - mid
+	pmull	v8.1q, v8.1d, v16.1d                     // GHASH final block - mid
+	eor	v11.16b, v11.16b, v21.16b                              // GHASH final block - low
+	eor	v10.16b, v10.16b, v8.16b                          // GHASH final block - mid
+	eor	v4.16b, v11.16b, v9.16b                                  // MODULO - karatsuba tidy up
+	fmov	d8, x21
+	eor	v10.16b, v10.16b, v4.16b                                  // MODULO - karatsuba tidy up
+	pmull	v7.1q, v9.1d, v8.1d                            // MODULO - top 64b align with mid
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor	v10.16b, v10.16b, v7.16b
+	eor	v10.16b, v10.16b, v9.16b                              // MODULO - fold into mid
+	pmull	v9.1q, v10.1d, v8.1d                           // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	st1	{ v5.16b}, [x2]                                        // store all 16B
+	eor	v11.16b, v11.16b, v9.16b
+	eor	v11.16b, v11.16b, v10.16b                             // MODULO - fold into low
+	ext	v11.16b, v11.16b, v11.16b, #8                             // Byte swap GHASH result
+	rev64	v11.16b, v11.16b                                           // Final Tag
+	mov	x0, x15
+	st1	{ v11.16b }, [x3]                                     // Store final tag
+	ldp	x19, x20, [sp, #16]
+	ldp	x21, x22, [sp, #32]
+	ldp	d8, d9, [sp, #64]
+	ldp	d10, d11, [sp, #80]
+	ldp	d12, d13, [sp, #96]
+	ldp	d14, d15, [sp, #112]
+	ldp	x29, x30, [sp], #224
+	AARCH64_VALIDATE_LINK_REGISTER
+	ret
+.size	aes_gcm_enc_kernel,.-aes_gcm_enc_kernel
+.globl	aes_gcm_dec_kernel
+.hidden	aes_gcm_dec_kernel
+.type	aes_gcm_dec_kernel,%function
+.align	4
+aes_gcm_dec_kernel:
+	AARCH64_SIGN_LINK_REGISTER
+	stp	x29, x30, [sp, #-224]!
+	mov	x29, sp
+	stp	x19, x20, [sp, #16]
+	ld1	{ v0.16b}, [x4]
+	mov	v1.16b, v0.16b
+	mov	v2.16b, v0.16b
+	mov	v3.16b, v0.16b
+	mov	x16, x4
+	mov	x8, x5
+	stp	x21, x22, [sp, #32]
+	stp	x23, x24, [sp, #48]
+	stp	d8, d9, [sp, #64]
+	stp	d10, d11, [sp, #80]
+	stp	d12, d13, [sp, #96]
+	stp	d14, d15, [sp, #112]
+	ldr	w17, [x8, #240]                                             // .Load number of AES rounds
+	add	x19, x8, x17, lsl #4                                    // borrow input_l1 for last key
+	ldp	x13, x14, [x19]                                        // load round N keys
+	ldr	q31, [x19, #-16]                                         // load round N-1 keys
+	add	x4, x0, x1, lsr #3                    // end_input_ptr
+	lsr	x5, x1, #3                               // byte_len
+	mov	x15, x5
+	ldr	w9, [x16, #12]                                           // .Load scalar 32-bit counter (CTR)
+	sub	x5, x5, #1                       // byte_len - 1
+	ldr	q18, [x8, #0]                                                   // load rk0
+	and	x5, x5, #0xffffffffffffffc0      // number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
+	add	x5, x5, x0
+	rev	w9, w9                                                   // Reverse it once for big-endian incrementing
+	uxtw	x10, w9                                                      // Zero extend reversed w9 into x10
+	str	q0,     [sp, #160]
+	str	q0,     [sp, #176]
+	str	q0,     [sp, #192]
+	str	q0,     [sp, #208]
+	rev	w20, w9
+	mov	v0.s[3], w20
+	add	w20, w9, #1
+	rev	w20, w20
+	mov	v1.s[3], w20
+	add	w20, w9, #2
+	rev	w20, w20
+	mov	v2.s[3], w20
+	add	w20, w9, #3
+	rev	w20, w20
+	mov	v3.s[3], w20
+	add	w20, w9, #4
+	rev	w20, w20
+	str	w20, [sp, #172]
+	add	w20, w9, #5
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w9, #6
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w9, #7
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w9, w9, #8
+    // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop
+	mov	x21, #0xc200000000000000
+	str	x21, [sp, #128]
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 0
+	ldp	q19, q20, [x8, #16]                                           // load rk1, rk2
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 0
+	ldp	q21, q22, [x8, #48]                                           // load rk3, rk4
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 0
+	ldp	q23, q24, [x8, #80]                                           // load rk5, rk6
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 0
+	ldp	q13, q14, [x6, #32]                                         // load h2, h3
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 1
+	ldp	q25, q26, [x8, #112]                                          // load rk7, rk8
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 1
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 2
+	ext	v14.16b, v14.16b, v14.16b, #8
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 2
+	ext	v13.16b, v13.16b, v13.16b, #8
+	ldr	q15, [x6, #80]                                               // load h4
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 2
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 2
+	ext	v15.16b, v15.16b, v15.16b, #8
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 3
+	ld1	{ v11.16b}, [x3]
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 3
+	ext	v11.16b, v11.16b, v11.16b, #8
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 3
+	rev64	v11.16b, v11.16b
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 3
+	trn2	v17.2d,  v14.2d,    v15.2d                                      // h4l | h3l
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 4
+	ldr	q12, [x6]                                                    // load h1
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 4
+	ext	v12.16b, v12.16b, v12.16b, #8
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 4
+	trn1	v9.2d, v14.2d,    v15.2d                                      // h4h | h3h
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 4
+	trn2	v16.2d,  v12.2d,    v13.2d                                      // h2l | h1l
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 5
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 5
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 5
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 5
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 6
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 6
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 6
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 6
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 7
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 7
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 7
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 7
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 8
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 8
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 8
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	b.lt	.Ldec_finish_first_blocks                                         // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	b.eq	.Ldec_finish_first_blocks                                         // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+.Ldec_finish_first_blocks:
+	ldr	q27, [x19]                                          // load rkN
+	cmp	x0, x5                                    // check if we have <= 4 blocks
+	eor	v17.16b, v17.16b, v9.16b                                   // h4k | h3k
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	trn1	v8.2d,    v12.2d,    v13.2d                                      // h2h | h1h
+	eor	v16.16b, v16.16b, v8.16b                                      // h2k | h1k
+	b.ge	.Ldec_tail                                                        // handle tail
+    // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions.
+    // This is because the final result of the AES block needs to be EORd with the final round key
+    // value (v30). This avoids several fmovs.
+	ldp	q6, q7, [x0, #32]                                  // AES blocks 2,3 load ciphertext
+	ldp	q4, q5, [x0], #64                                  // AES blocks 0,1 load ciphertext
+	eor	v0.16b, v4.16b, v0.16b
+	eor	v0.16b, v0.16b, v27.16b                           // AES block 0 - result
+	eor	v1.16b, v5.16b, v1.16b
+	eor	v1.16b, v1.16b, v27.16b                           // AES block 1 - result
+	eor	v2.16b, v6.16b, v2.16b
+	eor	v2.16b, v2.16b, v27.16b                           // AES block 2 - result
+	eor	v3.16b, v7.16b, v3.16b
+	eor	v3.16b, v3.16b, v27.16b                           // AES block 3 - result
+	st1	{ v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64  // AES blocks 0-3 - store result
+	ldr	q0,     [sp, #160]
+	ldr	q1,     [sp, #176]
+	ldr	q2,     [sp, #192]
+	ldr	q3,     [sp, #208]
+	rev	w20, w9
+	str	w20, [sp, #172]
+	add	w20, w9, #1
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w9, #2
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w9, #3
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w9, w9, #4
+	cmp	x0, x5                                    // check if we have <= 4 blocks
+	b.ge	.Ldec_prepretail                                                  // do prepretail
+.Ldec_main_loop:	//	main loop start
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 0
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 0
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 0
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 0
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 1
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 1
+	rev64	v4.16b, v4.16b                                             // GHASH block 4k
+	rev64	v5.16b, v5.16b                                             // GHASH block 4k+1
+	rev64	v6.16b, v6.16b                                             // GHASH block 4k+2
+	rev64	v7.16b, v7.16b                                             // GHASH block 4k+3
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 2
+	ext	v11.16b, v11.16b, v11.16b, #8                             // PRE 0
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 2
+	eor	v4.16b, v4.16b, v11.16b                                   // PRE 1
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 2
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 4k - low
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 2
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH block 4k - high
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 3
+	mov	d10, v17.d[1]                                                // GHASH block 4k - mid
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 3
+	mov	d8, v4.d[1]                                                   // GHASH block 4k - mid
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 3
+	eor	v8.8b, v8.8b, v4.8b                                           // GHASH block 4k - mid
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 3
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 4
+	pmull2	v4.1q, v5.2d, v14.2d                                        // GHASH block 4k+1 - high
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 4
+	eor	v9.16b, v9.16b, v4.16b                                    // GHASH block 4k+1 - high
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 4
+	mov	d4, v5.d[1]                                                   // GHASH block 4k+1 - mid
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 4
+	pmull	v10.1q, v8.1d, v10.1d                                     // GHASH block 4k - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 5
+	pmull	v8.1q, v5.1d, v14.1d                                         // GHASH block 4k+1 - low
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 5
+	eor	v4.8b, v4.8b, v5.8b                                           // GHASH block 4k+1 - mid
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 5
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 5
+	pmull	v5.1q, v6.1d, v13.1d                                         // GHASH block 4k+2 - low
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 6
+	eor	v11.16b, v11.16b, v8.16b
+	eor	v11.16b, v11.16b, v5.16b                                   // GHASH block 4k+1 - low & GHASH block 4k+2 - low
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 6
+	mov	d8, v6.d[1]                                                   // GHASH block 4k+2 - mid
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 6
+	eor	v8.8b, v8.8b, v6.8b                                           // GHASH block 4k+2 - mid
+	pmull	v4.1q, v4.1d, v17.1d                                         // GHASH block 4k+1 - mid
+	ins	v8.d[1], v8.d[0]                                                 // GHASH block 4k+2 - mid
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 6
+	pmull2	v28.1q, v6.2d, v13.2d                                   // GHASH block 4k+2 - high
+	mov	d6, v7.d[1]                                                   // GHASH block 4k+3 - mid
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 7
+	pmull2	v8.1q, v8.2d, v16.2d                                        // GHASH block 4k+2 - mid
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 7
+	pmull	v27.1q, v7.1d, v12.1d                                 // GHASH block 4k+3 - low
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 7
+	eor	v10.16b, v10.16b, v4.16b
+	eor	v10.16b, v10.16b, v8.16b                                   // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 7
+	pmull2	v5.1q, v7.2d, v12.2d                                        // GHASH block 4k+3 - high
+	eor	v6.8b, v6.8b, v7.8b                                           // GHASH block 4k+3 - mid
+	eor	v9.16b, v9.16b, v28.16b
+	eor	v9.16b, v9.16b, v5.16b                              // GHASH block 4k+2 - high & GHASH block 4k+3 - high
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 8
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 8
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 8
+	pmull	v6.1q, v6.1d, v16.1d                                         // GHASH block 4k+3 - mid
+	ldr	d8, [sp, #128]
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 8
+	pmull	v7.1q, v9.1d, v8.1d                           // MODULO - top 64b align with mid
+	eor	v10.16b, v10.16b, v6.16b
+	eor	v10.16b, v10.16b, v7.16b                                // GHASH block 4k+3 - mid & MODULO - fold into mid
+	eor	v11.16b, v11.16b, v27.16b                            // GHASH block 4k+3 - low
+	eor	v6.16b, v11.16b, v9.16b                                    // MODULO - karatsuba tidy up
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor	v10.16b, v10.16b, v6.16b
+	eor	v10.16b, v10.16b, v9.16b                                // MODULO - karatsuba tidy up & MODULO - fold into mid
+	pmull	v8.1q, v10.1d, v8.1d                    // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	eor	v11.16b, v11.16b, v8.16b
+	eor	v11.16b, v11.16b, v10.16b                      // MODULO - fold into low
+	cmp	w17, #12                                                     // setup flags for AES-128/192/256 check
+	b.lt	.Ldec_main_loop_continue                                          // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 10
+	b.eq	.Ldec_main_loop_continue                                          // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 12
+.Ldec_main_loop_continue:
+	ldr	q27, [x19]                                          // load rkN
+	ldp	q6, q7, [x0, #32]                                  // AES blocks 2,3 load ciphertext
+	ldp	q4, q5, [x0], #64                                  // AES blocks 0,1 load ciphertext
+	aese	v0.16b, v31.16b                                             // AES block 4k+4 - round N-1
+	eor	v0.16b, v4.16b, v0.16b
+	eor	v0.16b, v0.16b, v27.16b                           // AES block 4k+4 - result
+	aese	v1.16b, v31.16b                                             // AES block 4k+5 - round N-1
+	eor	v1.16b, v5.16b, v1.16b
+	eor	v1.16b, v1.16b, v27.16b                           // AES block 4k+5 - result
+	aese	v2.16b, v31.16b                                             // AES block 4k+6 - round N-1
+	eor	v2.16b, v6.16b, v2.16b
+	eor	v2.16b, v2.16b, v27.16b                           // AES block 4k+6 - result
+	aese	v3.16b, v31.16b                                             // AES block 4k+7 - round N-1
+	eor	v3.16b, v7.16b, v3.16b
+	eor	v3.16b, v3.16b, v27.16b                           // AES block 4k+7 - result
+	st1	{ v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64  // AES blocks 4k+4-7 - store result
+	ldr	q0,     [sp, #160]
+	ldr	q1,     [sp, #176]
+	ldr	q2,     [sp, #192]
+	ldr	q3,     [sp, #208]
+	rev	w20, w9
+	str	w20, [sp, #172]
+	add	w20, w9, #1
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w9, #2
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w9, #3
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w9, w9, #4
+	cmp	x0, x5
+	b.lt	.Ldec_main_loop
+.Ldec_prepretail:	//	PREPRETAIL
+	rev64	v4.16b, v4.16b                                             // GHASH block 0
+	rev64	v5.16b, v5.16b                                             // GHASH block 1
+	ext	v11.16b, v11.16b, v11.16b, #8                             // PRE 0
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 0
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 0
+	eor	v4.16b, v4.16b, v11.16b                                   // PRE 1
+	rev64	v6.16b, v6.16b                                             // GHASH block 2
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 1
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 0 - low
+	mov	d8, v4.d[1]                                                   // GHASH block 0 - mid
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH block 0 - high
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 0
+	mov	d10, v17.d[1]                                                // GHASH block 0 - mid
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 1
+	eor	v8.8b, v8.8b, v4.8b                                           // GHASH block 0 - mid
+	pmull2	v4.1q, v5.2d, v14.2d                                        // GHASH block 1 - high
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 1
+	rev64	v7.16b, v7.16b                                             // GHASH block 3
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 0
+	pmull	v10.1q, v8.1d, v10.1d                                     // GHASH block 0 - mid
+	eor	v9.16b, v9.16b, v4.16b                                    // GHASH block 1 - high
+	pmull	v8.1q, v5.1d, v14.1d                                         // GHASH block 1 - low
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 1
+	mov	d4, v5.d[1]                                                   // GHASH block 1 - mid
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 2
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 2
+	eor	v11.16b, v11.16b, v8.16b                                    // GHASH block 1 - low
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 2
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 3
+	mov	d8, v6.d[1]                                                   // GHASH block 2 - mid
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 2
+	eor	v4.8b, v4.8b, v5.8b                                           // GHASH block 1 - mid
+	pmull	v5.1q, v6.1d, v13.1d                                         // GHASH block 2 - low
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 4
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 3
+	eor	v8.8b, v8.8b, v6.8b                                           // GHASH block 2 - mid
+	pmull	v4.1q, v4.1d, v17.1d                                         // GHASH block 1 - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 5
+	eor	v11.16b, v11.16b, v5.16b                                    // GHASH block 2 - low
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 4
+	pmull2	v5.1q, v7.2d, v12.2d                                        // GHASH block 3 - high
+	eor	v10.16b, v10.16b, v4.16b                                    // GHASH block 1 - mid
+	pmull2	v4.1q, v6.2d, v13.2d                                        // GHASH block 2 - high
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 5
+	ins	v8.d[1], v8.d[0]                                                 // GHASH block 2 - mid
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 3
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 3
+	eor	v9.16b, v9.16b, v4.16b
+	eor	v9.16b, v9.16b, v5.16b                                   // GHASH block 2 - high & GHASH block 3 - high
+	pmull	v4.1q, v7.1d, v12.1d                                         // GHASH block 3 - low
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 4
+	mov	d6, v7.d[1]                                                   // GHASH block 3 - mid
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 4
+	pmull2	v8.1q, v8.2d, v16.2d                                        // GHASH block 2 - mid
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 5
+	eor	v6.8b, v6.8b, v7.8b                                           // GHASH block 3 - mid
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 5
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 6
+	eor	v10.16b, v10.16b, v8.16b                                    // GHASH block 2 - mid
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 6
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 6
+	movi	v8.8b, #0xc2
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 6
+	eor	v11.16b, v11.16b, v4.16b                                    // GHASH block 3 - low
+	pmull	v6.1q, v6.1d, v16.1d                                         // GHASH block 3 - mid
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 7
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 7
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 7
+	eor	v10.16b, v10.16b, v6.16b                                    // GHASH block 3 - mid
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 8
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 7
+	eor	v6.16b, v11.16b, v9.16b                                    // MODULO - karatsuba tidy up
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 8
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 8
+	shl	d8, d8, #56                                // mod_constant
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 8
+	cmp	w17, #12                                                     // setup flags for AES-128/192/256 check
+	b.lt	.Ldec_finish_prepretail                                           // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	b.eq	.Ldec_finish_prepretail                                           // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+.Ldec_finish_prepretail:
+	eor	v10.16b, v10.16b, v6.16b                                    // MODULO - karatsuba tidy up
+	pmull	v7.1q, v9.1d, v8.1d                           // MODULO - top 64b align with mid
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor	v10.16b, v10.16b, v7.16b
+	eor	v10.16b, v10.16b, v9.16b                             // MODULO - fold into mid
+	pmull	v8.1q, v10.1d, v8.1d                    // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	eor	v11.16b, v11.16b, v8.16b
+	eor	v11.16b, v11.16b, v10.16b                      // MODULO - fold into low
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+.Ldec_tail:	//	TAIL
+	sub	x5, x4, x0                    // main_end_input_ptr is number of bytes left to process
+	ld1	{ v5.16b}, [x0], #16                                    // AES block 0 - load ciphertext
+	eor	v0.16b, v5.16b, v0.16b                                    // AES block 0 - result
+	mov	x6, v0.d[0]                                             // AES block 0 - mov low
+	mov	x7, v0.d[1]                                             // AES block 0 - mov high
+	ext	v8.16b, v11.16b, v11.16b, #8                                // prepare final partial tag
+	eor	x6, x6, x13                                     // AES block 0 - round N low
+	eor	x7, x7, x14                                     // AES block 0 - round N high
+	cmp	x5, #48
+	b.gt	.Ldec_blocks_more_than_3
+	mov	v3.16b, v2.16b
+	movi	v10.8b, #0
+	movi	v11.8b, #0
+	movi	v9.8b, #0
+	mov	v2.16b, v1.16b
+	cmp	x5, #32
+	b.gt	.Ldec_blocks_more_than_2
+	mov	v3.16b, v1.16b
+	cmp	x5, #16
+	b.gt	.Ldec_blocks_more_than_1
+	b	.Ldec_blocks_less_than_1
+.Ldec_blocks_more_than_3:	//	blocks left >  3
+	rev64	v4.16b, v5.16b                                             // GHASH final-3 block
+	ld1	{ v5.16b}, [x0], #16                                    // AES final-2 block - load ciphertext
+	stp	x6, x7, [x2], #16                         // AES final-3 block  - store result
+	mov	d10, v17.d[1]                                                // GHASH final-3 block - mid
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	eor	v0.16b, v5.16b, v1.16b                                    // AES final-2 block - result
+	mov	d22, v4.d[1]                                                  // GHASH final-3 block - mid
+	mov	x6, v0.d[0]                                             // AES final-2 block - mov low
+	mov	x7, v0.d[1]                                             // AES final-2 block - mov high
+	eor	v22.8b, v22.8b, v4.8b                                       // GHASH final-3 block - mid
+	movi	v8.8b, #0                                                        // suppress further partial tag feed in
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH final-3 block - high
+	pmull	v10.1q, v22.1d, v10.1d                                   // GHASH final-3 block - mid
+	eor	x6, x6, x13                                     // AES final-2 block - round N low
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH final-3 block - low
+	eor	x7, x7, x14                                     // AES final-2 block - round N high
+.Ldec_blocks_more_than_2:	//	blocks left >  2
+	rev64	v4.16b, v5.16b                                             // GHASH final-2 block
+	ld1	{ v5.16b}, [x0], #16                                    // AES final-1 block - load ciphertext
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	stp	x6, x7, [x2], #16                         // AES final-2 block  - store result
+	eor	v0.16b, v5.16b, v2.16b                                    // AES final-1 block - result
+	mov	d22, v4.d[1]                                                  // GHASH final-2 block - mid
+	pmull	v21.1q, v4.1d, v14.1d                                         // GHASH final-2 block - low
+	pmull2	v20.1q, v4.2d, v14.2d                                       // GHASH final-2 block - high
+	eor	v22.8b, v22.8b, v4.8b                                       // GHASH final-2 block - mid
+	mov	x6, v0.d[0]                                             // AES final-1 block - mov low
+	mov	x7, v0.d[1]                                             // AES final-1 block - mov high
+	eor	v11.16b, v11.16b, v21.16b                                   // GHASH final-2 block - low
+	movi	v8.8b, #0                                                        // suppress further partial tag feed in
+	pmull	v22.1q, v22.1d, v17.1d                                     // GHASH final-2 block - mid
+	eor	v9.16b, v9.16b, v20.16b                                   // GHASH final-2 block - high
+	eor	x6, x6, x13                                     // AES final-1 block - round N low
+	eor	v10.16b, v10.16b, v22.16b                                  // GHASH final-2 block - mid
+	eor	x7, x7, x14                                     // AES final-1 block - round N high
+.Ldec_blocks_more_than_1:	//	blocks left >  1
+	stp	x6, x7, [x2], #16                         // AES final-1 block  - store result
+	rev64	v4.16b, v5.16b                                             // GHASH final-1 block
+	ld1	{ v5.16b}, [x0], #16                                    // AES final block - load ciphertext
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	movi	v8.8b, #0                                                        // suppress further partial tag feed in
+	mov	d22, v4.d[1]                                                  // GHASH final-1 block - mid
+	eor	v0.16b, v5.16b, v3.16b                                    // AES final block - result
+	pmull2	v20.1q, v4.2d, v13.2d                                        // GHASH final-1 block - high
+	eor	v22.8b, v22.8b, v4.8b                                       // GHASH final-1 block - mid
+	pmull	v21.1q, v4.1d, v13.1d                                         // GHASH final-1 block - low
+	mov	x6, v0.d[0]                                             // AES final block - mov low
+	ins	v22.d[1], v22.d[0]                                             // GHASH final-1 block - mid
+	mov	x7, v0.d[1]                                             // AES final block - mov high
+	pmull2	v22.1q, v22.2d, v16.2d                                    // GHASH final-1 block - mid
+	eor	x6, x6, x13                                     // AES final block - round N low
+	eor	v11.16b, v11.16b, v21.16b                                   // GHASH final-1 block - low
+	eor	v9.16b, v9.16b, v20.16b                                   // GHASH final-1 block - high
+	eor	v10.16b, v10.16b, v22.16b                                  // GHASH final-1 block - mid
+	eor	x7, x7, x14                                     // AES final block - round N high
+.Ldec_blocks_less_than_1:	//	blocks left <= 1
+	add	x10, x10, x1, lsr #7                                    // Calculate the updated counter based on the number of 16B chunks we processed
+	rev	w10, w10
+	str	w10, [x16, #12]                                              // store the updated counter
+	and	x1, x1, #127                                     // bit_length %= 128
+	mvn	x14, xzr                                                        // rkN_h = 0xffffffffffffffff
+	sub	x1, x1, #128                                     // bit_length -= 128
+	mvn	x13, xzr                                                        // rkN_l = 0xffffffffffffffff
+	ldp	x4, x5, [x2]                 // load existing bytes we need to not overwrite
+	neg	x1, x1                                           // bit_length = 128 - #bits in input (in range [1,128])
+	and	x1, x1, #127                                     // bit_length %= 128
+	lsr	x14, x14, x1                                        // rkN_h is mask for top 64b of last block
+	cmp	x1, #64
+	csel	x9, x13, x14, lt
+	csel	x10, x14, xzr, lt
+	fmov	d0, x9                                                   // ctr0b is mask for last block
+	and	x6, x6, x9
+	mov	v0.d[1], x10
+	bic	x4, x4, x9                            // mask out low existing bytes
+	bic	x5, x5, x10              // mask out high existing bytes
+	orr	x6, x6, x4
+	and	x7, x7, x10
+	orr	x7, x7, x5
+	and	v5.16b, v5.16b, v0.16b                                    // possibly partial last block has zeroes in highest bits
+	rev64	v4.16b, v5.16b                                             // GHASH final block
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	pmull	v21.1q, v4.1d, v12.1d                                         // GHASH final block - low
+	mov	d8, v4.d[1]                                                   // GHASH final block - mid
+	eor	v8.8b, v8.8b, v4.8b                                           // GHASH final block - mid
+	pmull2	v20.1q, v4.2d, v12.2d                                        // GHASH final block - high
+	pmull	v8.1q, v8.1d, v16.1d                                         // GHASH final block - mid
+	eor	v9.16b, v9.16b, v20.16b                                   // GHASH final block - high
+	eor	v11.16b, v11.16b, v21.16b                                   // GHASH final block - low
+	eor	v10.16b, v10.16b, v8.16b                                    // GHASH final block - mid
+	ldr	d8, [sp, #128]
+	eor	v6.16b, v11.16b, v9.16b                                    // MODULO - karatsuba tidy up
+	eor	v10.16b, v10.16b, v6.16b                                    // MODULO - karatsuba tidy up
+	pmull	v7.1q, v9.1d, v8.1d                           // MODULO - top 64b align with mid
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor	v10.16b, v10.16b, v7.16b                                 // MODULO - fold into mid
+	eor	v10.16b, v10.16b, v9.16b                                 // MODULO - fold into mid
+	pmull	v8.1q, v10.1d, v8.1d                    // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	eor	v11.16b, v11.16b, v8.16b                          // MODULO - fold into low
+	stp	x6, x7, [x2]
+	eor	v11.16b, v11.16b, v10.16b                                 // MODULO - fold into low
+	ext	v11.16b, v11.16b, v11.16b, #8
+	rev64	v11.16b, v11.16b                                           // Final Tag
+	mov	x0, x15
+	st1	{ v11.16b }, [x3]                                     // Store final tag
+	ldp	x19, x20, [sp, #16]
+	ldp	x21, x22, [sp, #32]
+	ldp	x23, x24, [sp, #48]
+	ldp	d8, d9, [sp, #64]
+	ldp	d10, d11, [sp, #80]
+	ldp	d12, d13, [sp, #96]
+	ldp	d14, d15, [sp, #112]
+	ldp	x29, x30, [sp], #224
+	AARCH64_VALIDATE_LINK_REGISTER
+	ret
+.size	aes_gcm_dec_kernel,.-aes_gcm_dec_kernel
+.globl	aes_gcm_enc_kernel_eor3
+.hidden	aes_gcm_enc_kernel_eor3
+.type	aes_gcm_enc_kernel_eor3,%function
+.align	4
+aes_gcm_enc_kernel_eor3:
+	AARCH64_SIGN_LINK_REGISTER
+	stp	x29, x30, [sp, #-224]!
+	mov	x29, sp
+	ld1	{ v0.16b}, [x4]                                                 // .Load initial counter block
+	stp	x19, x20, [sp, #16]
+	mov	v1.16b, v0.16b                                               // Initialize ctr1-3 from ctr0
+	mov	v2.16b, v0.16b
+	mov	v3.16b, v0.16b
+	mov	x16, x4                                                       // Pointer to counter block in memory
+	mov	x8, x5                                                            // Pointer to AES key schedule context
+	stp	x21, x22, [sp, #32]
+    // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel_eor3
+	stp	d8, d9, [sp, #64]                                                  // Save Neon registers
+	stp	d10, d11, [sp, #80]
+	stp	d12, d13, [sp, #96]
+	stp	d14, d15, [sp, #112]
+	ldr	w17, [x8, #240]                                             // .Load number of AES rounds
+	add	x7, x8, x17, lsl #4                                     // Calculate pointer to the last round key
+	ldp	x13, x14, [x7]                                         // load round N key (for final XOR)
+	ldr	q31, [x7, #-16]                                          // load round N-1 key
+	add	x4, x0, x1, lsr #3                    // Calculate end of input
+	lsr	x5, x1, #3                               // Total byte length
+	mov	x15, x5
+	ldr	w12, [x16, #12]                                            // .Load counter's low 32 bits
+	sub	x5, x5, #1                       // byte_len - 1
+	ldr	q18, [x8, #0]                                                   // load rk0
+	and	x5, x5, #0xffffffffffffffc0      // Align main loop end to a multiple of 64 bytes
+	add	x5, x5, x0
+	rev	w12, w12                                                     // Reverse for big-endian increment
+	uxtw	x10, w12                                                       // Zero extend reversed w12 into x10 for final counter update
+    // Pre-compute this value instead of using two instructions to reconstruct it every iteration
+	mov	x21, #0xc200000000000000                                // GHASH reduction constant
+	str	x21, [sp, #128]
+    // We maintain four copies of ctr values on the stack. Each loop iteration we
+    // store the updated ctr value to the last four bytes (e.g., 160 + 12).
+    // We then load the four values. This avoids a singificant number of
+    // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we
+    // calculate and store the values one iteration ahead so they have time to
+    // drain before we load them.
+	str	q0,     [sp, #160]                                  // Store base counter for block 0-3
+	str	q0,     [sp, #176]
+	str	q0,     [sp, #192]
+	str	q0,     [sp, #208]
+    // Since we need the values right away don't go through the stack this first
+    // time. Manually insert the incremented big-endian counter values.
+	rev	w20, w12
+	mov	v0.s[3], w20                                             // ctr0 + 0
+	add	w20, w12, #1
+	rev	w20, w20
+	mov	v1.s[3], w20                                             // ctr0 + 1
+	add	w20, w12, #2
+	rev	w20, w20
+	mov	v2.s[3], w20                                             // ctr0 + 2
+	add	w20, w12, #3
+	rev	w20, w20
+	mov	v3.s[3], w20                                             // ctr0 + 3
+    // Calculate the ctr values for the *next* (not current) group of four
+    // blocks. Store the incremented parts to the stack.
+	add	w20, w12, #4
+	rev	w20, w20
+	str	w20, [sp, #172]                             // ctr0 + 4 for next iter
+	add	w20, w12, #5
+	rev	w20, w20
+	str	w20, [sp, #188]                             // ctr0 + 5 for next iter
+	add	w20, w12, #6
+	rev	w20, w20
+	str	w20, [sp, #204]                             // ctr0 + 6 for next iter
+	add	w20, w12, #7
+	rev	w20, w20
+	str	w20, [sp, #220]                             // ctr0 + 7 for next iter
+	add	w12, w12, #8                                                 // Advance counter past these two sets
+    // --- Start AES for first 4 blocks ---
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 0
+	ldp	q19, q20, [x8, #16]                                           // load rk1, rk2
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 0
+	ldp	q21, q22, [x8, #48]                                           // load rk3, rk4
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 0
+	ldp	q23, q24, [x8, #80]                                           // load rk5, rk6
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 0
+	ldp	q13, q14, [x6, #32]                                         // load H2, H3 (GHASH keys)
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 1
+	ldp	q25, q26, [x8, #112]                                          // load rk7, rk8
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 1
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 2
+	ext	v14.16b, v14.16b, v14.16b, #8                                      // Byte swap H3 for GHASH
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 2
+	ext	v13.16b, v13.16b, v13.16b, #8                                      // Byte swap H2 for GHASH
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 2
+	ldr	q15, [x6, #80]                                               // load H4
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 2
+	ext	v15.16b, v15.16b, v15.16b, #8                                      // Byte swap H4 for GHASH
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 3
+	ld1	{ v11.16b}, [x3]                                      // .Load initial GHASH accumulator (T)
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 3
+	ext	v11.16b, v11.16b, v11.16b, #8                             // Byte swap T for GHASH
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 3
+	rev64	v11.16b, v11.16b                                           // Correct byte order within 64-bit lanes
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 3
+	trn2	v17.2d,  v14.2d,    v15.2d                                      // Karatsuba key: H4_low | H3_low
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 4
+	ldr	q12, [x6]                                                    // load H1
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 4
+	ext	v12.16b, v12.16b, v12.16b, #8                                      // Byte swap H1 for GHASH
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 4
+	trn1	v9.2d, v14.2d,    v15.2d                                      // Karatsuba key: H4_high | H3_high
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 4
+	trn2	v16.2d,  v12.2d,    v13.2d                                      // Karatsuba key: H2_low | H1_low
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 5
+	ldr	q30, [x7]                                                // Preload round N key for final EOR
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 5
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 5
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 5
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 6
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 6
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 6
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 6
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 7
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 7
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 7
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 7
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 8
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 8
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 8
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	b.lt	.Lenc_finish_first_blocks_eor3                                         // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	b.eq	.Lenc_finish_first_blocks_eor3                                         // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+.Lenc_finish_first_blocks_eor3:
+	cmp	x0, x5                                    // check if we have <= 4 blocks to process in the tail
+	eor	v17.16b, v17.16b, v9.16b                                   // Karatsuba key: H3^H4
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	trn1	v8.2d,  v12.2d, v13.2d                                 // Karatsuba key: H2_high | H1_high
+	eor	v16.16b, v16.16b, v8.16b                            // Karatsuba key: H1^H2
+	b.ge	.Lenc_tail_eor3                                                        // handle tail if no more full 4-block sets
+	ldp	q6, q7, [x0, #32]                                  // AES blocks 2,3 load plaintext
+	ldp	q4, q5, [x0], #64                                  // AES blocks 0,1 load plaintext
+    // Compute and store first 4 ciphertext blocks
+	eor3	v4.16b, v4.16b, v30.16b, v0.16b                                // AES block 0 - result = PT ^ AES(ctr0)
+	eor3	v5.16b, v5.16b, v30.16b, v1.16b                                // AES block 1 - result = PT ^ AES(ctr1)
+	eor3	v6.16b, v6.16b, v30.16b, v2.16b                                // AES block 2 - result = PT ^ AES(ctr2)
+	eor3	v7.16b, v7.16b, v30.16b, v3.16b                                // AES block 3 - result = PT ^ AES(ctr3)
+	st1	{ v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64  // AES blocks 0-3 - store result
+    // Load counter values for the second iteration from the stack
+	ldp	q0, q1, [sp, #160]
+	ldp	q2, q3, [sp, #192]
+    // Prepare and store counter values for the third iteration
+	rev	w20, w12
+	str	w20, [sp, #172]                             // ctr + 8
+	add	w20, w12, #1
+	rev	w20, w20
+	str	w20, [sp, #188]                             // ctr + 9
+	add	w20, w12, #2
+	rev	w20, w20
+	str	w20, [sp, #204]                             // ctr + 10
+	add	w20, w12, #3
+	rev	w20, w20
+	str	w20, [sp, #220]                             // ctr + 11
+	add	w12, w12, #4                                                 // Advance counter base
+	cmp	x0, x5                                    // check if we have <= 4 blocks remaining
+	b.ge	.Lenc_prepretail_eor3                                                  // go to prepretail if < 2 full loops left
+.Lenc_main_loop_eor3:	//	main loop start (processes 4 blocks per iteration)
+    // --- AES Pipeline for blocks 4k+4 to 4k+7 ---
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 0
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 0
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 0
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 0
+	ldr	d8, [sp, #128]                      // .Load GHASH reduction constant
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 1
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 1
+    // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 ---
+	rev64	v4.16b, v4.16b                                             // GHASH block 4k - Byte swap CT
+	rev64	v5.16b, v5.16b                                             // GHASH block 4k+1 - Byte swap CT
+	rev64	v6.16b, v6.16b                                             // GHASH block 4k+2 - Byte swap CT
+	rev64	v7.16b, v7.16b                                             // GHASH block 4k+3 - Byte swap CT
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 2
+	ext	v11.16b, v11.16b, v11.16b, #8                             // GHASH - prepare acc for XOR
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 2
+	eor	v4.16b, v4.16b, v11.16b                                   // GHASH block 4k - Y_i = CT_i ^ Y_{i-1}
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 2
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 4k - low
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 2
+	pmull2	v9.1q, v6.2d, v13.2d                                     // GHASH block 4k+2 - high
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 3
+	mov	d10, v17.d[1]                                                // GHASH block 4k - mid Karatsuba key
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 3
+	mov	d20, v4.d[1]                                             // GHASH block 4k - mid
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 3
+	eor	v20.8b, v20.8b, v4.8b                               // GHASH block 4k - mid
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 3
+	mov	d21, v5.d[1]                                             // GHASH block 4k+1 - mid
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 4
+	eor	v21.8b, v21.8b, v5.8b                               // GHASH block 4k+1 - mid
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 4
+	pmull	v10.1q, v20.1d, v10.1d                               // GHASH block 4k - mid
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 4
+	pmull	v21.1q, v21.1d, v17.1d                             // GHASH block 4k+1 - mid
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 4
+	eor	v10.16b, v10.16b, v21.16b                              // GHASH block 4k+1 - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 5
+	ext	v22.16b, v22.16b, v6.16b, #8                        // GHASH block 4k+2 - mid
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 5
+	eor	v22.16b, v22.16b, v6.16b                            // GHASH block 4k+2 - mid
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 5
+	pmull2	v22.1q, v22.2d, v16.2d                            // GHASH block 4k+2 - mid
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 5
+	mov	d23, v7.d[1]                                             // GHASH block 4k+3 - mid
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 6
+	eor	v23.8b, v23.8b, v7.8b                               // GHASH block 4k+3 - mid
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 6
+	pmull	v23.1q, v23.1d, v16.1d                             // GHASH block 4k+3 - mid
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 6
+	eor3	v10.16b, v10.16b, v22.16b, v23.16b                       // GHASH block 4k+2/3 - mid
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 6
+	pmull2	v22.1q, v4.2d, v15.2d                                  // GHASH block 4k - high
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 7
+	pmull2	v21.1q, v7.2d, v12.2d                                  // GHASH block 4k+3 - high
+	eor	v22.16b, v22.16b, v21.16b                        // GHASH block 4k+3 - high
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 7
+	pmull2	v23.1q, v5.2d, v14.2d                                  // GHASH block 4k+1 - high
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 7
+	pmull	v21.1q, v6.1d, v13.1d                                   // GHASH block 4k+2 - low
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 7
+	pmull	v20.1q, v5.1d, v14.1d                                   // GHASH block 4k+1 - low
+	eor3	v9.16b, v9.16b, v22.16b, v23.16b                       // GHASH block 4k/1/2/3 - high
+	pmull	v22.1q, v7.1d, v12.1d                                   // GHASH block 4k+3 - low
+	ldp	q6, q7, [x0, #32]
+	ldp	q4, q5, [x0], #64
+	eor	v20.16b, v20.16b, v21.16b                        // GHASH block 4k+1 - low
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 4k+4 - round 8
+	eor3	v11.16b, v11.16b, v22.16b, v20.16b                       // GHASH block 4k/1/2/3 - low
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 4k+5 - round 8
+	eor3	v10.16b, v10.16b, v9.16b, v11.16b                             // MODULO - karatsuba tidy up
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 4k+6 - round 8
+	pmull	v20.1q, v9.1d, v8.1d                        // MODULO - top 64b align with mid
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 4k+7 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	b.lt	.Lenc_main_loop_continue_eor3                                          // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 10
+	b.eq	.Lenc_main_loop_continue_eor3                                          // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 12
+.Lenc_main_loop_continue_eor3:
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor3	v10.16b, v10.16b, v20.16b, v9.16b                          // MODULO - fold into mid
+	pmull	v9.1q, v10.1d, v8.1d                           // MODULO - mid 64b align with low
+	ext	v20.16b, v10.16b, v10.16b, #8                          // MODULO - other mid alignment
+	eor3	v11.16b, v9.16b, v11.16b, v20.16b                          // MODULO - fold into low
+	aese	v0.16b, v31.16b                                             // AES block 4k+4 - round N-1
+	eor3	v4.16b, v4.16b, v30.16b, v0.16b                                // AES block 4k+4 - result
+	aese	v1.16b, v31.16b                                             // AES block 4k+5 - round N-1
+	eor3	v5.16b, v5.16b, v30.16b, v1.16b                                // AES block 4k+5 - result
+	aese	v2.16b, v31.16b                                             // AES block 4k+6 - round N-1
+	eor3	v6.16b, v6.16b, v30.16b, v2.16b                                // AES block 4k+6 - result
+	aese	v3.16b, v31.16b                                             // AES block 4k+7 - round N-1
+	eor3	v7.16b, v7.16b, v30.16b, v3.16b                                // AES block 4k+7 - result
+	ldp	q0, q1, [sp, #160]
+	ldp	q2, q3, [sp, #192]
+    // We used these registers as temporaries above so reload the RKs.
+	ldp	q20, q21, [x8, #32]                                           // load rk2, rk3
+	ldp	q22, q23, [x8, #64]                                           // load rk4, rk5
+	st1	{ v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64  // AES blocks 4k+4-7 - store result
+	rev	w20, w12
+	str	w20, [sp, #172]
+	add	w20, w12, #1
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w12, #2
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w12, #3
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w12, w12, #4
+	cmp	x0, x5                                    // .LOOP_eor3 CONTROL
+	b.lt	.Lenc_main_loop_eor3
+.Lenc_prepretail_eor3:	//	PREPRETAIL
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 0
+	rev64	v6.16b, v6.16b                                             // GHASH block 2
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 0
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 0
+	rev64	v4.16b, v4.16b                                             // GHASH block 0
+	ext	v11.16b, v11.16b, v11.16b, #8                             // PRE 0
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 1
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 1
+	eor	v4.16b, v4.16b, v11.16b                                   // PRE 1
+	rev64	v5.16b, v5.16b                                             // GHASH block 1
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 2
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 0
+	mov	d10, v17.d[1]                                                // GHASH block 0 - mid Karatsuba key
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 1
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 0 - low
+	mov	d8, v4.d[1]                                         // GHASH block 0 - mid
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH block 0 - high
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 3
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 2
+	eor	v8.8b, v8.8b, v4.8b                       // GHASH block 0 - mid
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 2
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 1
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 3
+	pmull	v10.1q, v8.1d, v10.1d                           // GHASH block 0 - mid
+	pmull2	v4.1q, v5.2d, v14.2d                                      // GHASH block 1 - high
+	pmull	v8.1q, v5.1d, v14.1d                               // GHASH block 1 - low
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 2
+	eor	v9.16b, v9.16b, v4.16b                                  // GHASH block 1 - high
+	mov	d4, v5.d[1]                                                 // GHASH block 1 - mid
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 3
+	eor	v11.16b, v11.16b, v8.16b                          // GHASH block 1 - low
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 3
+	eor	v4.8b, v4.8b, v5.8b                                       // GHASH block 1 - mid
+	mov	d8, v6.d[1]                                         // GHASH block 2 - mid
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 4
+	rev64	v7.16b, v7.16b                                             // GHASH block 3
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 4
+	pmull	v4.1q, v4.1d, v17.1d                                     // GHASH block 1 - mid
+	eor	v8.8b, v8.8b, v6.8b                       // GHASH block 2 - mid
+	pmull	v5.1q, v6.1d, v13.1d                                       // GHASH block 2 - low
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 5
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 4
+	eor	v10.16b, v10.16b, v4.16b                                  // GHASH block 1 - mid
+	pmull2	v4.1q, v6.2d, v13.2d                                      // GHASH block 2 - high
+	eor	v11.16b, v11.16b, v5.16b                                  // GHASH block 2 - low
+	ins	v8.d[1], v8.d[0]                             // GHASH block 2 - mid
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 5
+	eor	v9.16b, v9.16b, v4.16b                                  // GHASH block 2 - high
+	mov	d4, v7.d[1]                                                 // GHASH block 3 - mid
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 4
+	pmull2	v8.1q, v8.2d, v16.2d                    // GHASH block 2 - mid
+	eor	v4.8b, v4.8b, v7.8b                                       // GHASH block 3 - mid
+	pmull2	v5.1q, v7.2d, v12.2d                                      // GHASH block 3 - high
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 5
+	pmull	v4.1q, v4.1d, v16.1d                                     // GHASH block 3 - mid
+	eor	v10.16b, v10.16b, v8.16b                          // GHASH block 2 - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 5
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 6
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 6
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 6
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 6
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 7
+	eor	v9.16b, v9.16b, v5.16b                                  // GHASH block 3 - high
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 7
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 7
+	ldr	d8, [sp, #128]
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 8
+	eor	v10.16b, v10.16b, v4.16b                                  // GHASH block 3 - mid
+	pmull	v6.1q, v7.1d, v12.1d                                       // GHASH block 3 - low
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 8
+	eor	v11.16b, v11.16b, v6.16b                                  // GHASH block 3 - low
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 7
+	eor	v10.16b, v10.16b, v9.16b                                 // karatsuba tidy up
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 8
+	pmull	v4.1q, v9.1d, v8.1d
+	ext	v9.16b, v9.16b, v9.16b, #8
+	eor	v10.16b, v10.16b, v11.16b
+	b.lt	.Lenc_finish_prepretail_eor3                                           // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	b.eq	.Lenc_finish_prepretail_eor3                                           // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+.Lenc_finish_prepretail_eor3:
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	eor3	v10.16b, v10.16b, v4.16b, v9.16b
+	pmull	v4.1q, v10.1d, v8.1d
+	ext	v10.16b, v10.16b, v10.16b, #8
+	eor3	v11.16b, v11.16b, v4.16b, v10.16b
+.Lenc_tail_eor3:	//	TAIL: Process remaining 0 to 3 blocks
+	ext	v8.16b, v11.16b, v11.16b, #8                      // Save current GHASH state for partial tag feed-in
+	sub	x5, x4, x0                    // main_end_input_ptr is number of bytes left to process
+	ldp	x6, x7, [x0], #16                            // AES block 0 - load plaintext
+	eor	x6, x6, x13                                       // AES block 0 - round N low
+	eor	x7, x7, x14                                       // AES block 0 - round N high
+	cmp	x5, #48
+	fmov	d4, x6                                                 // AES block 0 - mov low
+	fmov	v4.d[1], x7                                             // AES block 0 - mov high
+	eor	v5.16b, v4.16b, v0.16b                                    // AES block 0 - result
+	b.gt	.Lenc_blocks_more_than_3_eor3
+	cmp	x5, #32
+	mov	v3.16b, v2.16b
+	movi	v11.8b, #0
+	movi	v9.8b, #0
+	mov	v2.16b, v1.16b
+	movi	v10.8b, #0
+	b.gt	.Lenc_blocks_more_than_2_eor3
+	mov	v3.16b, v1.16b
+	cmp	x5, #16
+	b.gt	.Lenc_blocks_more_than_1_eor3
+	b	.Lenc_blocks_less_than_1_eor3
+.Lenc_blocks_more_than_3_eor3:	//	blocks left >  3
+	st1	{ v5.16b}, [x2], #16                                   // AES final-2 block - store result
+	ldp	x6, x7, [x0], #16                            // AES final-2 block - load input low & high
+	rev64	v4.16b, v5.16b                                             // GHASH final-3 block
+	eor	x6, x6, x13                                       // AES final-2 block - round N low
+	eor	v4.16b, v4.16b, v8.16b                            // feed in partial tag
+	eor	x7, x7, x14                                       // AES final-2 block - round N high
+	mov	d22, v4.d[1]                                             // GHASH final-3 block - mid
+	fmov	d5, x6                                                 // AES final-2 block - mov low
+	fmov	v5.d[1], x7                                             // AES final-2 block - mov high
+	eor	v22.8b, v22.8b, v4.8b                               // GHASH final-3 block - mid
+	movi	v8.8b, #0                                              // suppress further partial tag feed in
+	mov	d10, v17.d[1]                                                // GHASH final-3 block - mid
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH final-3 block - low
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH final-3 block - high
+	pmull	v10.1q, v22.1d, v10.1d                               // GHASH final-3 block - mid
+	eor	v5.16b, v5.16b, v1.16b                                    // AES final-2 block - result
+.Lenc_blocks_more_than_2_eor3:	//	blocks left >  2
+	st1	{ v5.16b}, [x2], #16                                   // AES final-2 block - store result
+	ldp	x6, x7, [x0], #16                            // AES final-1 block - load input low & high
+	rev64	v4.16b, v5.16b                                             // GHASH final-2 block
+	eor	x6, x6, x13                                       // AES final-1 block - round N low
+	eor	v4.16b, v4.16b, v8.16b                            // feed in partial tag
+	fmov	d5, x6                                                 // AES final-1 block - mov low
+	eor	x7, x7, x14                                       // AES final-1 block - round N high
+	fmov	v5.d[1], x7                                             // AES final-1 block - mov high
+	movi	v8.8b, #0                                              // suppress further partial tag feed in
+	pmull2	v20.1q, v4.2d, v14.2d                                  // GHASH final-2 block - high
+	mov	d22, v4.d[1]                                             // GHASH final-2 block - mid
+	pmull	v21.1q, v4.1d, v14.1d                                   // GHASH final-2 block - low
+	eor	v22.8b, v22.8b, v4.8b                               // GHASH final-2 block - mid
+	eor	v5.16b, v5.16b, v2.16b                                    // AES final-1 block - result
+	eor	v9.16b, v9.16b, v20.16b                              // GHASH final-2 block - high
+	pmull	v22.1q, v22.1d, v17.1d                             // GHASH final-2 block - mid
+	eor	v11.16b, v11.16b, v21.16b                              // GHASH final-2 block - low
+	eor	v10.16b, v10.16b, v22.16b                              // GHASH final-2 block - mid
+.Lenc_blocks_more_than_1_eor3:	//	blocks left >  1
+	st1	{ v5.16b}, [x2], #16                                   // AES final-1 block - store result
+	rev64	v4.16b, v5.16b                                             // GHASH final-1 block: Byte Swap CT
+	ldp	x6, x7, [x0], #16                            // AES final block - load plaintext
+	eor	v4.16b, v4.16b, v8.16b                            // Feed in partial tag
+	movi	v8.8b, #0                                              // Clear for next block
+	eor	x6, x6, x13                                       // AES final block - round N low
+	mov	d22, v4.d[1]                                             // GHASH final-1 block - mid
+	pmull2	v20.1q, v4.2d, v13.2d                                  // GHASH final-1 block - high
+	eor	x7, x7, x14                                       // AES final block - round N high
+	eor	v22.8b, v22.8b, v4.8b                               // GHASH final-1 block - mid
+	eor	v9.16b, v9.16b, v20.16b                              // GHASH final-1 block - high
+	ins	v22.d[1], v22.d[0]                                     // GHASH final-1 block - mid
+	fmov	d5, x6                                                 // AES final block - mov low
+	fmov	v5.d[1], x7                                             // AES final block - mov high
+	pmull2	v22.1q, v22.2d, v16.2d                            // GHASH final-1 block - mid
+	pmull	v21.1q, v4.1d, v13.1d                                   // GHASH final-1 block - low
+	eor	v5.16b, v5.16b, v3.16b                                    // AES final block - result
+	eor	v10.16b, v10.16b, v22.16b                              // GHASH final-1 block - mid
+	eor	v11.16b, v11.16b, v21.16b                              // GHASH final-1 block - low
+.Lenc_blocks_less_than_1_eor3:	//	.Last partial block handling
+	add	x10, x10, x1, lsr #7                                    // Calculate the updated counter based on the number of 16B chunks we processed
+	rev	w10, w10
+	str	w10, [x16, #12]                                              // store the updated counter
+	and	x1, x1, #127                                     // bit_length %= 128
+	mvn	x13, xzr                                                        // Mask for low 64 bits
+	sub	x1, x1, #128                                     //
+	neg	x1, x1                                           // Valid bits in the last block (1-128)
+	ldr	q18, [x2]                                  // .Load destination for merging
+	mvn	x14, xzr                                                        // Mask for high 64 bits
+	and	x1, x1, #127                                     // bit_length %= 128
+	lsr	x14, x14, x1                                        // rkN_h is mask for top 64b of last block
+	cmp	x1, #64
+	csel	x6, x13, x14, lt
+	csel	x7, x14, xzr, lt
+	fmov	d0, x6                                                 // ctr0d is mask for last block
+	fmov	v0.d[1], x7
+	and	v5.16b, v5.16b, v0.16b                                    // Mask out unused bits of the last CT block
+	rev64	v4.16b, v5.16b                                             // GHASH final block - byte swap
+	eor	v4.16b, v4.16b, v8.16b                            // Feed in partial tag
+	bif	v5.16b, v18.16b, v0.16b                        // Bitwise Insert: merge with existing data at output_ptr
+	pmull2	v20.1q, v4.2d, v12.2d                                  // GHASH final block - high
+	mov	d8, v4.d[1]                                         // GHASH final block - mid
+	pmull	v21.1q, v4.1d, v12.1d                                   // GHASH final block - low
+	eor	v9.16b, v9.16b, v20.16b                              // GHASH final block - high
+	eor	v8.8b, v8.8b, v4.8b                       // GHASH final block - mid
+	pmull	v8.1q, v8.1d, v16.1d                     // GHASH final block - mid
+	eor	v11.16b, v11.16b, v21.16b                              // GHASH final block - low
+	eor	v10.16b, v10.16b, v8.16b                          // GHASH final block - mid
+	eor	v4.16b, v11.16b, v9.16b                                  // MODULO - karatsuba tidy up
+	fmov	d8, x21
+	eor	v10.16b, v10.16b, v4.16b                                  // MODULO - karatsuba tidy up
+	pmull	v7.1q, v9.1d, v8.1d                            // MODULO - top 64b align with mid
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor3	v10.16b, v10.16b, v7.16b, v9.16b                              // MODULO - fold into mid
+	pmull	v9.1q, v10.1d, v8.1d                           // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	st1	{ v5.16b}, [x2]                                        // store all 16B
+	eor3	v11.16b, v11.16b, v9.16b, v10.16b                             // MODULO - fold into low
+	ext	v11.16b, v11.16b, v11.16b, #8                             // Byte swap GHASH result
+	rev64	v11.16b, v11.16b                                           // Final Tag
+	mov	x0, x15
+	st1	{ v11.16b }, [x3]                                     // Store final tag
+	ldp	x19, x20, [sp, #16]
+	ldp	x21, x22, [sp, #32]
+	ldp	d8, d9, [sp, #64]
+	ldp	d10, d11, [sp, #80]
+	ldp	d12, d13, [sp, #96]
+	ldp	d14, d15, [sp, #112]
+	ldp	x29, x30, [sp], #224
+	AARCH64_VALIDATE_LINK_REGISTER
+	ret
+.size	aes_gcm_enc_kernel_eor3,.-aes_gcm_enc_kernel_eor3
+.globl	aes_gcm_dec_kernel_eor3
+.hidden	aes_gcm_dec_kernel_eor3
+.type	aes_gcm_dec_kernel_eor3,%function
+.align	4
+aes_gcm_dec_kernel_eor3:
+	AARCH64_SIGN_LINK_REGISTER
+	stp	x29, x30, [sp, #-224]!
+	mov	x29, sp
+	stp	x19, x20, [sp, #16]
+	ld1	{ v0.16b}, [x4]
+	mov	v1.16b, v0.16b
+	mov	v2.16b, v0.16b
+	mov	v3.16b, v0.16b
+	mov	x16, x4
+	mov	x8, x5
+	stp	x21, x22, [sp, #32]
+	stp	x23, x24, [sp, #48]
+	stp	d8, d9, [sp, #64]
+	stp	d10, d11, [sp, #80]
+	stp	d12, d13, [sp, #96]
+	stp	d14, d15, [sp, #112]
+	ldr	w17, [x8, #240]                                             // .Load number of AES rounds
+	add	x19, x8, x17, lsl #4                                    // borrow input_l1 for last key
+	ldp	x13, x14, [x19]                                        // load round N keys
+	ldr	q31, [x19, #-16]                                         // load round N-1 keys
+	add	x4, x0, x1, lsr #3                    // end_input_ptr
+	lsr	x5, x1, #3                               // byte_len
+	mov	x15, x5
+	ldr	w9, [x16, #12]                                           // .Load scalar 32-bit counter (CTR)
+	sub	x5, x5, #1                       // byte_len - 1
+	ldr	q18, [x8, #0]                                                   // load rk0
+	and	x5, x5, #0xffffffffffffffc0      // number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
+	add	x5, x5, x0
+	rev	w9, w9                                                   // Reverse it once for big-endian incrementing
+	uxtw	x10, w9                                                      // Zero extend reversed w9 into x10
+	str	q0,     [sp, #160]
+	str	q0,     [sp, #176]
+	str	q0,     [sp, #192]
+	str	q0,     [sp, #208]
+	rev	w20, w9
+	mov	v0.s[3], w20
+	add	w20, w9, #1
+	rev	w20, w20
+	mov	v1.s[3], w20
+	add	w20, w9, #2
+	rev	w20, w20
+	mov	v2.s[3], w20
+	add	w20, w9, #3
+	rev	w20, w20
+	mov	v3.s[3], w20
+	add	w20, w9, #4
+	rev	w20, w20
+	str	w20, [sp, #172]
+	add	w20, w9, #5
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w9, #6
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w9, #7
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w9, w9, #8
+    // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop
+	mov	x21, #0xc200000000000000
+	str	x21, [sp, #128]
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 0
+	ldp	q19, q20, [x8, #16]                                           // load rk1, rk2
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 0
+	ldp	q21, q22, [x8, #48]                                           // load rk3, rk4
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 0
+	ldp	q23, q24, [x8, #80]                                           // load rk5, rk6
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 0
+	ldp	q13, q14, [x6, #32]                                         // load h2, h3
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 1
+	ldp	q25, q26, [x8, #112]                                          // load rk7, rk8
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 1
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 2
+	ext	v14.16b, v14.16b, v14.16b, #8
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 2
+	ext	v13.16b, v13.16b, v13.16b, #8
+	ldr	q15, [x6, #80]                                               // load h4
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 2
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 2
+	ext	v15.16b, v15.16b, v15.16b, #8
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 3
+	ld1	{ v11.16b}, [x3]
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 3
+	ext	v11.16b, v11.16b, v11.16b, #8
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 3
+	rev64	v11.16b, v11.16b
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 3
+	trn2	v17.2d,  v14.2d,    v15.2d                                      // h4l | h3l
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 4
+	ldr	q12, [x6]                                                    // load h1
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 4
+	ext	v12.16b, v12.16b, v12.16b, #8
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 4
+	trn1	v9.2d, v14.2d,    v15.2d                                      // h4h | h3h
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 4
+	trn2	v16.2d,  v12.2d,    v13.2d                                      // h2l | h1l
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 5
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 5
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 5
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 5
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 6
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 6
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 6
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 6
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 7
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 7
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 7
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 7
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b           // AES block 0 - round 8
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b           // AES block 1 - round 8
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b           // AES block 2 - round 8
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b           // AES block 3 - round 8
+	cmp	x17, #12                                                       // setup flags for AES-128/192/256 check
+	b.lt	.Ldec_finish_first_blocks_eor3                                         // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	b.eq	.Ldec_finish_first_blocks_eor3                                         // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+.Ldec_finish_first_blocks_eor3:
+	ldr	q27, [x19]                                          // load rkN
+	cmp	x0, x5                                    // check if we have <= 4 blocks
+	eor	v17.16b, v17.16b, v9.16b                                   // h4k | h3k
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	trn1	v8.2d,    v12.2d,    v13.2d                                      // h2h | h1h
+	eor	v16.16b, v16.16b, v8.16b                                      // h2k | h1k
+	b.ge	.Ldec_tail_eor3                                                        // handle tail
+    // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions.
+    // This is because the final result of the AES block needs to be EORd with the final round key
+    // value (v30). This avoids several fmovs.
+	ldp	q6, q7, [x0, #32]                                  // AES blocks 2,3 load ciphertext
+	ldp	q4, q5, [x0], #64                                  // AES blocks 0,1 load ciphertext
+	eor3	v0.16b, v4.16b, v0.16b, v27.16b                           // AES block 0 - result
+	eor3	v1.16b, v5.16b, v1.16b, v27.16b                           // AES block 1 - result
+	eor3	v2.16b, v6.16b, v2.16b, v27.16b                           // AES block 2 - result
+	eor3	v3.16b, v7.16b, v3.16b, v27.16b                           // AES block 3 - result
+	st1	{ v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64  // AES blocks 0-3 - store result
+	ldr	q0,     [sp, #160]
+	ldr	q1,     [sp, #176]
+	ldr	q2,     [sp, #192]
+	ldr	q3,     [sp, #208]
+	rev	w20, w9
+	str	w20, [sp, #172]
+	add	w20, w9, #1
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w9, #2
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w9, #3
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w9, w9, #4
+	cmp	x0, x5                                    // check if we have <= 4 blocks
+	b.ge	.Ldec_prepretail_eor3                                                  // do prepretail
+.Ldec_main_loop_eor3:	//	main loop start
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 0
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 0
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 0
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 0
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 1
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 1
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 1
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 1
+	rev64	v4.16b, v4.16b                                             // GHASH block 4k
+	rev64	v5.16b, v5.16b                                             // GHASH block 4k+1
+	rev64	v6.16b, v6.16b                                             // GHASH block 4k+2
+	rev64	v7.16b, v7.16b                                             // GHASH block 4k+3
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 2
+	ext	v11.16b, v11.16b, v11.16b, #8                             // PRE 0
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 2
+	eor	v4.16b, v4.16b, v11.16b                                   // PRE 1
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 2
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 4k - low
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 2
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH block 4k - high
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 3
+	mov	d10, v17.d[1]                                                // GHASH block 4k - mid
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 3
+	mov	d8, v4.d[1]                                                   // GHASH block 4k - mid
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 3
+	eor	v8.8b, v8.8b, v4.8b                                           // GHASH block 4k - mid
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 3
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 4
+	pmull2	v4.1q, v5.2d, v14.2d                                        // GHASH block 4k+1 - high
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 4
+	eor	v9.16b, v9.16b, v4.16b                                    // GHASH block 4k+1 - high
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 4
+	mov	d4, v5.d[1]                                                   // GHASH block 4k+1 - mid
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 4
+	pmull	v10.1q, v8.1d, v10.1d                                     // GHASH block 4k - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 5
+	pmull	v8.1q, v5.1d, v14.1d                                         // GHASH block 4k+1 - low
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 5
+	eor	v4.8b, v4.8b, v5.8b                                           // GHASH block 4k+1 - mid
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 5
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 5
+	pmull	v5.1q, v6.1d, v13.1d                                         // GHASH block 4k+2 - low
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 6
+	eor3	v11.16b, v11.16b, v8.16b, v5.16b                                   // GHASH block 4k+1 - low & GHASH block 4k+2 - low
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 6
+	mov	d8, v6.d[1]                                                   // GHASH block 4k+2 - mid
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 6
+	eor	v8.8b, v8.8b, v6.8b                                           // GHASH block 4k+2 - mid
+	pmull	v4.1q, v4.1d, v17.1d                                         // GHASH block 4k+1 - mid
+	ins	v8.d[1], v8.d[0]                                                 // GHASH block 4k+2 - mid
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 6
+	pmull2	v28.1q, v6.2d, v13.2d                                   // GHASH block 4k+2 - high
+	mov	d6, v7.d[1]                                                   // GHASH block 4k+3 - mid
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 7
+	pmull2	v8.1q, v8.2d, v16.2d                                        // GHASH block 4k+2 - mid
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 7
+	pmull	v27.1q, v7.1d, v12.1d                                 // GHASH block 4k+3 - low
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 7
+	eor3	v10.16b, v10.16b, v4.16b, v8.16b                                   // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 7
+	pmull2	v5.1q, v7.2d, v12.2d                                        // GHASH block 4k+3 - high
+	eor	v6.8b, v6.8b, v7.8b                                           // GHASH block 4k+3 - mid
+	eor3	v9.16b, v9.16b, v28.16b, v5.16b                              // GHASH block 4k+2 - high & GHASH block 4k+3 - high
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b          // AES block 4k+4 - round 8
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b          // AES block 4k+5 - round 8
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b          // AES block 4k+6 - round 8
+	pmull	v6.1q, v6.1d, v16.1d                                         // GHASH block 4k+3 - mid
+	ldr	d8, [sp, #128]
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b          // AES block 4k+7 - round 8
+	pmull	v7.1q, v9.1d, v8.1d                           // MODULO - top 64b align with mid
+	eor3	v10.16b, v10.16b, v6.16b, v7.16b                                // GHASH block 4k+3 - mid & MODULO - fold into mid
+	eor	v11.16b, v11.16b, v27.16b                            // GHASH block 4k+3 - low
+	eor	v6.16b, v11.16b, v9.16b                                    // MODULO - karatsuba tidy up
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor3	v10.16b, v10.16b, v6.16b, v9.16b                                // MODULO - karatsuba tidy up & MODULO - fold into mid
+	pmull	v8.1q, v10.1d, v8.1d                    // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	eor3	v11.16b, v11.16b, v8.16b, v10.16b                      // MODULO - fold into low
+	cmp	w17, #12                                                     // setup flags for AES-128/192/256 check
+	b.lt	.Ldec_main_loop_continue_eor3                                          // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 10
+	b.eq	.Ldec_main_loop_continue_eor3                                          // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 4k+4 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 4k+5 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 4k+6 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 4k+7 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 4k+4 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 4k+5 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 4k+6 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 4k+7 - round 12
+.Ldec_main_loop_continue_eor3:
+	ldr	q27, [x19]                                          // load rkN
+	ldp	q6, q7, [x0, #32]                                  // AES blocks 2,3 load ciphertext
+	ldp	q4, q5, [x0], #64                                  // AES blocks 0,1 load ciphertext
+	aese	v0.16b, v31.16b                                             // AES block 4k+4 - round N-1
+	eor3	v0.16b, v4.16b, v0.16b, v27.16b                           // AES block 4k+4 - result
+	aese	v1.16b, v31.16b                                             // AES block 4k+5 - round N-1
+	eor3	v1.16b, v5.16b, v1.16b, v27.16b                           // AES block 4k+5 - result
+	aese	v2.16b, v31.16b                                             // AES block 4k+6 - round N-1
+	eor3	v2.16b, v6.16b, v2.16b, v27.16b                           // AES block 4k+6 - result
+	aese	v3.16b, v31.16b                                             // AES block 4k+7 - round N-1
+	eor3	v3.16b, v7.16b, v3.16b, v27.16b                           // AES block 4k+7 - result
+	st1	{ v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64  // AES blocks 4k+4-7 - store result
+	ldr	q0,     [sp, #160]
+	ldr	q1,     [sp, #176]
+	ldr	q2,     [sp, #192]
+	ldr	q3,     [sp, #208]
+	rev	w20, w9
+	str	w20, [sp, #172]
+	add	w20, w9, #1
+	rev	w20, w20
+	str	w20, [sp, #188]
+	add	w20, w9, #2
+	rev	w20, w20
+	str	w20, [sp, #204]
+	add	w20, w9, #3
+	rev	w20, w20
+	str	w20, [sp, #220]
+	add	w9, w9, #4
+	cmp	x0, x5
+	b.lt	.Ldec_main_loop_eor3
+.Ldec_prepretail_eor3:	//	PREPRETAIL
+	rev64	v4.16b, v4.16b                                             // GHASH block 0
+	rev64	v5.16b, v5.16b                                             // GHASH block 1
+	ext	v11.16b, v11.16b, v11.16b, #8                             // PRE 0
+	aese	v0.16b, v18.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 0
+	aese	v1.16b, v18.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 0
+	eor	v4.16b, v4.16b, v11.16b                                   // PRE 1
+	rev64	v6.16b, v6.16b                                             // GHASH block 2
+	aese	v1.16b, v19.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 1
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH block 0 - low
+	mov	d8, v4.d[1]                                                   // GHASH block 0 - mid
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH block 0 - high
+	aese	v2.16b, v18.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 0
+	mov	d10, v17.d[1]                                                // GHASH block 0 - mid
+	aese	v0.16b, v19.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 1
+	eor	v8.8b, v8.8b, v4.8b                                           // GHASH block 0 - mid
+	pmull2	v4.1q, v5.2d, v14.2d                                        // GHASH block 1 - high
+	aese	v2.16b, v19.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 1
+	rev64	v7.16b, v7.16b                                             // GHASH block 3
+	aese	v3.16b, v18.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 0
+	pmull	v10.1q, v8.1d, v10.1d                                     // GHASH block 0 - mid
+	eor	v9.16b, v9.16b, v4.16b                                    // GHASH block 1 - high
+	pmull	v8.1q, v5.1d, v14.1d                                         // GHASH block 1 - low
+	aese	v3.16b, v19.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 1
+	mov	d4, v5.d[1]                                                   // GHASH block 1 - mid
+	aese	v0.16b, v20.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 2
+	aese	v1.16b, v20.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 2
+	eor	v11.16b, v11.16b, v8.16b                                    // GHASH block 1 - low
+	aese	v2.16b, v20.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 2
+	aese	v0.16b, v21.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 3
+	mov	d8, v6.d[1]                                                   // GHASH block 2 - mid
+	aese	v3.16b, v20.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 2
+	eor	v4.8b, v4.8b, v5.8b                                           // GHASH block 1 - mid
+	pmull	v5.1q, v6.1d, v13.1d                                         // GHASH block 2 - low
+	aese	v0.16b, v22.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 4
+	aese	v3.16b, v21.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 3
+	eor	v8.8b, v8.8b, v6.8b                                           // GHASH block 2 - mid
+	pmull	v4.1q, v4.1d, v17.1d                                         // GHASH block 1 - mid
+	aese	v0.16b, v23.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 5
+	eor	v11.16b, v11.16b, v5.16b                                    // GHASH block 2 - low
+	aese	v3.16b, v22.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 4
+	pmull2	v5.1q, v7.2d, v12.2d                                        // GHASH block 3 - high
+	eor	v10.16b, v10.16b, v4.16b                                    // GHASH block 1 - mid
+	pmull2	v4.1q, v6.2d, v13.2d                                        // GHASH block 2 - high
+	aese	v3.16b, v23.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 5
+	ins	v8.d[1], v8.d[0]                                                 // GHASH block 2 - mid
+	aese	v2.16b, v21.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 3
+	aese	v1.16b, v21.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 3
+	eor3	v9.16b, v9.16b, v4.16b, v5.16b                                   // GHASH block 2 - high & GHASH block 3 - high
+	pmull	v4.1q, v7.1d, v12.1d                                         // GHASH block 3 - low
+	aese	v2.16b, v22.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 4
+	mov	d6, v7.d[1]                                                   // GHASH block 3 - mid
+	aese	v1.16b, v22.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 4
+	pmull2	v8.1q, v8.2d, v16.2d                                        // GHASH block 2 - mid
+	aese	v2.16b, v23.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 5
+	eor	v6.8b, v6.8b, v7.8b                                           // GHASH block 3 - mid
+	aese	v1.16b, v23.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 5
+	aese	v3.16b, v24.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 6
+	eor	v10.16b, v10.16b, v8.16b                                    // GHASH block 2 - mid
+	aese	v2.16b, v24.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 6
+	aese	v0.16b, v24.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 6
+	movi	v8.8b, #0xc2
+	aese	v1.16b, v24.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 6
+	eor	v11.16b, v11.16b, v4.16b                                    // GHASH block 3 - low
+	pmull	v6.1q, v6.1d, v16.1d                                         // GHASH block 3 - mid
+	aese	v3.16b, v25.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 7
+	aese	v1.16b, v25.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 7
+	aese	v0.16b, v25.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 7
+	eor	v10.16b, v10.16b, v6.16b                                    // GHASH block 3 - mid
+	aese	v3.16b, v26.16b
+	aesmc	v3.16b, v3.16b          // AES block 3 - round 8
+	aese	v2.16b, v25.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 7
+	eor	v6.16b, v11.16b, v9.16b                                    // MODULO - karatsuba tidy up
+	aese	v1.16b, v26.16b
+	aesmc	v1.16b, v1.16b          // AES block 1 - round 8
+	aese	v0.16b, v26.16b
+	aesmc	v0.16b, v0.16b          // AES block 0 - round 8
+	shl	d8, d8, #56                                // mod_constant
+	aese	v2.16b, v26.16b
+	aesmc	v2.16b, v2.16b          // AES block 2 - round 8
+	cmp	w17, #12                                                     // setup flags for AES-128/192/256 check
+	b.lt	.Ldec_finish_prepretail_eor3                                           // branch if AES-128
+	ldp	q27, q28, [x8, #144]                               // load rk9, rk10
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 9
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 9
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 9
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 9
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 10
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 10
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 10
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 10
+	b.eq	.Ldec_finish_prepretail_eor3                                           // branch if AES-192
+	ldp	q27, q28, [x8, #176]                               // load rk11, rk12
+	aese	v0.16b, v27.16b
+	aesmc	v0.16b, v0.16b    // AES block 0 - round 11
+	aese	v1.16b, v27.16b
+	aesmc	v1.16b, v1.16b    // AES block 1 - round 11
+	aese	v2.16b, v27.16b
+	aesmc	v2.16b, v2.16b    // AES block 2 - round 11
+	aese	v3.16b, v27.16b
+	aesmc	v3.16b, v3.16b    // AES block 3 - round 11
+	aese	v0.16b, v28.16b
+	aesmc	v0.16b, v0.16b       // AES block 0 - round 12
+	aese	v1.16b, v28.16b
+	aesmc	v1.16b, v1.16b       // AES block 1 - round 12
+	aese	v2.16b, v28.16b
+	aesmc	v2.16b, v2.16b       // AES block 2 - round 12
+	aese	v3.16b, v28.16b
+	aesmc	v3.16b, v3.16b       // AES block 3 - round 12
+.Ldec_finish_prepretail_eor3:
+	eor	v10.16b, v10.16b, v6.16b                                    // MODULO - karatsuba tidy up
+	pmull	v7.1q, v9.1d, v8.1d                           // MODULO - top 64b align with mid
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor3	v10.16b, v10.16b, v7.16b, v9.16b                             // MODULO - fold into mid
+	pmull	v8.1q, v10.1d, v8.1d                    // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	eor3	v11.16b, v11.16b, v8.16b, v10.16b                      // MODULO - fold into low
+	aese	v1.16b, v31.16b                                             // AES block 1 - round N-1
+	aese	v0.16b, v31.16b                                             // AES block 0 - round N-1
+	aese	v3.16b, v31.16b                                             // AES block 3 - round N-1
+	aese	v2.16b, v31.16b                                             // AES block 2 - round N-1
+.Ldec_tail_eor3:	//	TAIL
+	sub	x5, x4, x0                    // main_end_input_ptr is number of bytes left to process
+	ld1	{ v5.16b}, [x0], #16                                    // AES block 0 - load ciphertext
+	eor	v0.16b, v5.16b, v0.16b                                    // AES block 0 - result
+	mov	x6, v0.d[0]                                             // AES block 0 - mov low
+	mov	x7, v0.d[1]                                             // AES block 0 - mov high
+	ext	v8.16b, v11.16b, v11.16b, #8                                // prepare final partial tag
+	eor	x6, x6, x13                                     // AES block 0 - round N low
+	eor	x7, x7, x14                                     // AES block 0 - round N high
+	cmp	x5, #48
+	b.gt	.Ldec_blocks_more_than_3_eor3
+	mov	v3.16b, v2.16b
+	movi	v10.8b, #0
+	movi	v11.8b, #0
+	movi	v9.8b, #0
+	mov	v2.16b, v1.16b
+	cmp	x5, #32
+	b.gt	.Ldec_blocks_more_than_2_eor3
+	mov	v3.16b, v1.16b
+	cmp	x5, #16
+	b.gt	.Ldec_blocks_more_than_1_eor3
+	b	.Ldec_blocks_less_than_1_eor3
+.Ldec_blocks_more_than_3_eor3:	//	blocks left >  3
+	rev64	v4.16b, v5.16b                                             // GHASH final-3 block
+	ld1	{ v5.16b}, [x0], #16                                    // AES final-2 block - load ciphertext
+	stp	x6, x7, [x2], #16                         // AES final-3 block  - store result
+	mov	d10, v17.d[1]                                                // GHASH final-3 block - mid
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	eor	v0.16b, v5.16b, v1.16b                                    // AES final-2 block - result
+	mov	d22, v4.d[1]                                                  // GHASH final-3 block - mid
+	mov	x6, v0.d[0]                                             // AES final-2 block - mov low
+	mov	x7, v0.d[1]                                             // AES final-2 block - mov high
+	eor	v22.8b, v22.8b, v4.8b                                       // GHASH final-3 block - mid
+	movi	v8.8b, #0                                                        // suppress further partial tag feed in
+	pmull2	v9.1q, v4.2d, v15.2d                                     // GHASH final-3 block - high
+	pmull	v10.1q, v22.1d, v10.1d                                   // GHASH final-3 block - mid
+	eor	x6, x6, x13                                     // AES final-2 block - round N low
+	pmull	v11.1q, v4.1d, v15.1d                                      // GHASH final-3 block - low
+	eor	x7, x7, x14                                     // AES final-2 block - round N high
+.Ldec_blocks_more_than_2_eor3:	//	blocks left >  2
+	rev64	v4.16b, v5.16b                                             // GHASH final-2 block
+	ld1	{ v5.16b}, [x0], #16                                    // AES final-1 block - load ciphertext
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	stp	x6, x7, [x2], #16                         // AES final-2 block  - store result
+	eor	v0.16b, v5.16b, v2.16b                                    // AES final-1 block - result
+	mov	d22, v4.d[1]                                                  // GHASH final-2 block - mid
+	pmull	v21.1q, v4.1d, v14.1d                                         // GHASH final-2 block - low
+	pmull2	v20.1q, v4.2d, v14.2d                                       // GHASH final-2 block - high
+	eor	v22.8b, v22.8b, v4.8b                                       // GHASH final-2 block - mid
+	mov	x6, v0.d[0]                                             // AES final-1 block - mov low
+	mov	x7, v0.d[1]                                             // AES final-1 block - mov high
+	eor	v11.16b, v11.16b, v21.16b                                   // GHASH final-2 block - low
+	movi	v8.8b, #0                                                        // suppress further partial tag feed in
+	pmull	v22.1q, v22.1d, v17.1d                                     // GHASH final-2 block - mid
+	eor	v9.16b, v9.16b, v20.16b                                   // GHASH final-2 block - high
+	eor	x6, x6, x13                                     // AES final-1 block - round N low
+	eor	v10.16b, v10.16b, v22.16b                                  // GHASH final-2 block - mid
+	eor	x7, x7, x14                                     // AES final-1 block - round N high
+.Ldec_blocks_more_than_1_eor3:	//	blocks left >  1
+	stp	x6, x7, [x2], #16                         // AES final-1 block  - store result
+	rev64	v4.16b, v5.16b                                             // GHASH final-1 block
+	ld1	{ v5.16b}, [x0], #16                                    // AES final block - load ciphertext
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	movi	v8.8b, #0                                                        // suppress further partial tag feed in
+	mov	d22, v4.d[1]                                                  // GHASH final-1 block - mid
+	eor	v0.16b, v5.16b, v3.16b                                    // AES final block - result
+	pmull2	v20.1q, v4.2d, v13.2d                                        // GHASH final-1 block - high
+	eor	v22.8b, v22.8b, v4.8b                                       // GHASH final-1 block - mid
+	pmull	v21.1q, v4.1d, v13.1d                                         // GHASH final-1 block - low
+	mov	x6, v0.d[0]                                             // AES final block - mov low
+	ins	v22.d[1], v22.d[0]                                             // GHASH final-1 block - mid
+	mov	x7, v0.d[1]                                             // AES final block - mov high
+	pmull2	v22.1q, v22.2d, v16.2d                                    // GHASH final-1 block - mid
+	eor	x6, x6, x13                                     // AES final block - round N low
+	eor	v11.16b, v11.16b, v21.16b                                   // GHASH final-1 block - low
+	eor	v9.16b, v9.16b, v20.16b                                   // GHASH final-1 block - high
+	eor	v10.16b, v10.16b, v22.16b                                  // GHASH final-1 block - mid
+	eor	x7, x7, x14                                     // AES final block - round N high
+.Ldec_blocks_less_than_1_eor3:	//	blocks left <= 1
+	add	x10, x10, x1, lsr #7                                    // Calculate the updated counter based on the number of 16B chunks we processed
+	rev	w10, w10
+	str	w10, [x16, #12]                                              // store the updated counter
+	and	x1, x1, #127                                     // bit_length %= 128
+	mvn	x14, xzr                                                        // rkN_h = 0xffffffffffffffff
+	sub	x1, x1, #128                                     // bit_length -= 128
+	mvn	x13, xzr                                                        // rkN_l = 0xffffffffffffffff
+	ldp	x4, x5, [x2]                 // load existing bytes we need to not overwrite
+	neg	x1, x1                                           // bit_length = 128 - #bits in input (in range [1,128])
+	and	x1, x1, #127                                     // bit_length %= 128
+	lsr	x14, x14, x1                                        // rkN_h is mask for top 64b of last block
+	cmp	x1, #64
+	csel	x9, x13, x14, lt
+	csel	x10, x14, xzr, lt
+	fmov	d0, x9                                                   // ctr0b is mask for last block
+	and	x6, x6, x9
+	mov	v0.d[1], x10
+	bic	x4, x4, x9                            // mask out low existing bytes
+	bic	x5, x5, x10              // mask out high existing bytes
+	orr	x6, x6, x4
+	and	x7, x7, x10
+	orr	x7, x7, x5
+	and	v5.16b, v5.16b, v0.16b                                    // possibly partial last block has zeroes in highest bits
+	rev64	v4.16b, v5.16b                                             // GHASH final block
+	eor	v4.16b, v4.16b, v8.16b                                      // feed in partial tag
+	pmull	v21.1q, v4.1d, v12.1d                                         // GHASH final block - low
+	mov	d8, v4.d[1]                                                   // GHASH final block - mid
+	eor	v8.8b, v8.8b, v4.8b                                           // GHASH final block - mid
+	pmull2	v20.1q, v4.2d, v12.2d                                        // GHASH final block - high
+	pmull	v8.1q, v8.1d, v16.1d                                         // GHASH final block - mid
+	eor	v9.16b, v9.16b, v20.16b                                   // GHASH final block - high
+	eor	v11.16b, v11.16b, v21.16b                                   // GHASH final block - low
+	eor	v10.16b, v10.16b, v8.16b                                    // GHASH final block - mid
+	ldr	d8, [sp, #128]
+	eor	v6.16b, v11.16b, v9.16b                                    // MODULO - karatsuba tidy up
+	eor	v10.16b, v10.16b, v6.16b                                    // MODULO - karatsuba tidy up
+	pmull	v7.1q, v9.1d, v8.1d                           // MODULO - top 64b align with mid
+	ext	v9.16b, v9.16b, v9.16b, #8                             // MODULO - other top alignment
+	eor	v10.16b, v10.16b, v7.16b                                 // MODULO - fold into mid
+	eor	v10.16b, v10.16b, v9.16b                                 // MODULO - fold into mid
+	pmull	v8.1q, v10.1d, v8.1d                    // MODULO - mid 64b align with low
+	ext	v10.16b, v10.16b, v10.16b, #8                             // MODULO - other mid alignment
+	eor	v11.16b, v11.16b, v8.16b                          // MODULO - fold into low
+	stp	x6, x7, [x2]
+	eor	v11.16b, v11.16b, v10.16b                                 // MODULO - fold into low
+	ext	v11.16b, v11.16b, v11.16b, #8
+	rev64	v11.16b, v11.16b                                           // Final Tag
+	mov	x0, x15
+	st1	{ v11.16b }, [x3]                                     // Store final tag
+	ldp	x19, x20, [sp, #16]
+	ldp	x21, x22, [sp, #32]
+	ldp	x23, x24, [sp, #48]
+	ldp	d8, d9, [sp, #64]
+	ldp	d10, d11, [sp, #80]
+	ldp	d12, d13, [sp, #96]
+	ldp	d14, d15, [sp, #112]
+	ldp	x29, x30, [sp], #224
+	AARCH64_VALIDATE_LINK_REGISTER
+	ret
+.size	aes_gcm_dec_kernel_eor3,.-aes_gcm_dec_kernel_eor3
+#endif  // __ARM_MAX_ARCH__ >= 8
+#endif  // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__ELF__)