visaljs 0.0.2 → 0.0.3

package/runtime/third_party/boringssl/crypto/cipher/e_aesgcmsiv.cc

@@ -0,0 +1,918 @@
+// Copyright 2017 The BoringSSL Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <openssl/aead.h>
+
+#include <assert.h>
+
+#include <openssl/cipher.h>
+#include <openssl/crypto.h>
+#include <openssl/err.h>
+#include <openssl/span.h>
+
+#include "../fipsmodule/aes/internal.h"
+#include "../fipsmodule/cipher/internal.h"
+#include "../internal.h"
+#include "internal.h"
+
+
+using namespace bssl;
+
+#define EVP_AEAD_AES_GCM_SIV_NONCE_LEN 12
+#define EVP_AEAD_AES_GCM_SIV_TAG_LEN 16
+
+namespace {
+void inc_counter(uint8_t tag[16], uint32_t by) {
+  CRYPTO_store_u32_le(tag, CRYPTO_load_u32_le(tag) + by);
+}
+
+#if defined(AES_GCM_SIV_ASM)
+
+// The assembly code assumes 8-byte alignment of the EVP_AEAD_CTX's state, and
+// aligns to 16 bytes itself.
+static_assert(sizeof(((EVP_AEAD_CTX *)nullptr)->state) >=
+                  sizeof(struct aead_aes_gcm_siv_asm_ctx) + 8,
+              "AEAD state is too small");
+static_assert(alignof(union evp_aead_ctx_st_state) >= 8,
+              "AEAD state has insufficient alignment");
+
+// asm_ctx_from_ctx returns a 16-byte aligned context pointer from `ctx`.
+struct aead_aes_gcm_siv_asm_ctx *asm_ctx_from_ctx(const EVP_AEAD_CTX *ctx) {
+  // ctx->state must already be 8-byte aligned. Thus, at most, we may need to
+  // add eight to align it to 16 bytes.
+  const uintptr_t offset = ((uintptr_t)&ctx->state) & 8;
+  return (struct aead_aes_gcm_siv_asm_ctx *)(&ctx->state.opaque[offset]);
+}
+
+int aead_aes_gcm_siv_asm_init(EVP_AEAD_CTX *ctx, const uint8_t *key,
+                              size_t key_len, size_t tag_len) {
+  const size_t key_bits = key_len * 8;
+
+  if (key_bits != 128 && key_bits != 256) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_KEY_LENGTH);
+    return 0;  // EVP_AEAD_CTX_init should catch this.
+  }
+
+  if (tag_len == EVP_AEAD_DEFAULT_TAG_LENGTH) {
+    tag_len = EVP_AEAD_AES_GCM_SIV_TAG_LEN;
+  }
+
+  if (tag_len != EVP_AEAD_AES_GCM_SIV_TAG_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TAG_TOO_LARGE);
+    return 0;
+  }
+
+  struct aead_aes_gcm_siv_asm_ctx *gcm_siv_ctx = asm_ctx_from_ctx(ctx);
+  assert((((uintptr_t)gcm_siv_ctx) & 15) == 0);
+
+  if (key_bits == 128) {
+    aes128gcmsiv_aes_ks(key, &gcm_siv_ctx->key[0]);
+    gcm_siv_ctx->is_128_bit = 1;
+  } else {
+    aes256gcmsiv_aes_ks(key, &gcm_siv_ctx->key[0]);
+    gcm_siv_ctx->is_128_bit = 0;
+  }
+
+  ctx->tag_len = tag_len;
+
+  return 1;
+}
+
+void aead_aes_gcm_siv_asm_cleanup(EVP_AEAD_CTX *ctx) {}
+
+// gcm_siv_asm_polyval evaluates POLYVAL at `auth_key` on the given plaintext
+// and AD. The result is written to `out_tag`, which must be 16-byte aligned.
+void gcm_siv_asm_polyval(uint8_t out_tag[16], Span<const CRYPTO_IOVEC> iovecs,
+                         Span<const CRYPTO_IVEC> aadvecs,
+                         const uint8_t auth_key[16], const uint8_t nonce[12]) {
+  OPENSSL_memset(out_tag, 0, 16);
+  const size_t ad_len = bssl::iovec::TotalLength(aadvecs);
+  const size_t ad_blocks = ad_len / 16;
+  const size_t in_len = bssl::iovec::TotalLength(iovecs);
+  const size_t in_blocks = in_len / 16;
+  int htable_init = 0;
+  alignas(16) uint8_t htable[16 * 8];
+
+  if (ad_blocks > 8 || in_blocks > 8) {
+    htable_init = 1;
+    aesgcmsiv_htable_init(htable, auth_key);
+  }
+
+  auto f_whole = [&](const uint8_t *in, size_t len) {
+    if (htable_init) {
+      aesgcmsiv_htable_polyval(htable, in, len, out_tag);
+    } else {
+      aesgcmsiv_polyval_horner(out_tag, auth_key, in, len / AES_BLOCK_SIZE);
+    }
+    return true;
+  };
+
+  auto f_final = [&](const uint8_t *in, size_t len) {
+    size_t len_whole = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+    if (len_whole != 0) {
+      f_whole(in, len_whole);
+      in += len_whole;
+      len -= len_whole;
+    }
+    if (len != 0) {
+      uint8_t pad_buf[AES_BLOCK_SIZE];
+      OPENSSL_memcpy(pad_buf, in, len);
+      OPENSSL_memset(pad_buf + len, 0, AES_BLOCK_SIZE - len);
+      aesgcmsiv_polyval_horner(out_tag, auth_key, pad_buf, 1);
+    }
+    return true;
+  };
+
+  bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/false>(
+      aadvecs, f_whole, f_final);
+
+  bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/false>(
+      iovecs, f_whole, f_final);
+
+  uint8_t length_block[16];
+  CRYPTO_store_u64_le(length_block, ad_len * 8);
+  CRYPTO_store_u64_le(length_block + 8, in_len * 8);
+  aesgcmsiv_polyval_horner(out_tag, auth_key, length_block, 1);
+
+  for (size_t i = 0; i < 12; i++) {
+    out_tag[i] ^= nonce[i];
+  }
+
+  out_tag[15] &= 0x7f;
+}
+
+// aead_aes_gcm_siv_asm_crypt_last_block handles the encryption/decryption
+// (same thing in CTR mode) of the final block of a plaintext/ciphertext. It
+// writes `total_in_len` & 15 bytes to `out_last_block`, based on an initial
+// counter derived from `tag`.
+void aead_aes_gcm_siv_asm_crypt_last_block(
+    int is_128_bit, uint8_t *out_last_block, const uint8_t *in_last_block,
+    size_t total_in_len, const uint8_t tag[16],
+    const struct aead_aes_gcm_siv_asm_ctx *enc_key_expanded) {
+  alignas(16) uint8_t counter[16];
+  OPENSSL_memcpy(&counter, tag, sizeof(counter));
+  counter[15] |= 0x80;
+  inc_counter(counter, static_cast<uint32_t>(total_in_len / 16));
+
+  if (is_128_bit) {
+    aes128gcmsiv_ecb_enc_block(counter, counter, enc_key_expanded);
+  } else {
+    aes256gcmsiv_ecb_enc_block(counter, counter, enc_key_expanded);
+  }
+
+  const size_t last_bytes_len = total_in_len & 15;
+  for (size_t i = 0; i < last_bytes_len; i++) {
+    out_last_block[i] = in_last_block[i] ^ counter[i];
+  }
+}
+
+// aead_aes_gcm_siv_kdf calculates the record encryption and authentication
+// keys given the `nonce`.
+void aead_aes_gcm_siv_kdf(int is_128_bit,
+                          const struct aead_aes_gcm_siv_asm_ctx *gcm_siv_ctx,
+                          uint64_t out_record_auth_key[2],
+                          uint64_t out_record_enc_key[4],
+                          const uint8_t nonce[12]) {
+  alignas(16) uint8_t padded_nonce[16] = {0};
+  OPENSSL_memcpy(padded_nonce, nonce, 12);
+
+  alignas(16) uint64_t key_material[12];
+  if (is_128_bit) {
+    aes128gcmsiv_kdf(padded_nonce, key_material, &gcm_siv_ctx->key[0]);
+    out_record_enc_key[0] = key_material[4];
+    out_record_enc_key[1] = key_material[6];
+  } else {
+    aes256gcmsiv_kdf(padded_nonce, key_material, &gcm_siv_ctx->key[0]);
+    out_record_enc_key[0] = key_material[4];
+    out_record_enc_key[1] = key_material[6];
+    out_record_enc_key[2] = key_material[8];
+    out_record_enc_key[3] = key_material[10];
+  }
+
+  out_record_auth_key[0] = key_material[0];
+  out_record_auth_key[1] = key_material[2];
+}
+
+int aead_aes_gcm_siv_asm_sealv(const EVP_AEAD_CTX *ctx,
+                               Span<const CRYPTO_IOVEC> iovecs,
+                               Span<uint8_t> out_tag, size_t *out_tag_len,
+                               Span<const uint8_t> nonce,
+                               Span<const CRYPTO_IVEC> aadvecs) {
+  const struct aead_aes_gcm_siv_asm_ctx *gcm_siv_ctx = asm_ctx_from_ctx(ctx);
+  const size_t in_len = bssl::iovec::TotalLength(iovecs);
+  const uint64_t in_len_64 = in_len;
+  const size_t ad_len = bssl::iovec::TotalLength(aadvecs);
+  const uint64_t ad_len_64 = ad_len;
+
+  if (in_len_64 > (UINT64_C(1) << 36) || ad_len_64 >= (UINT64_C(1) << 61)) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE);
+    return 0;
+  }
+
+  if (out_tag.size() < EVP_AEAD_AES_GCM_SIV_TAG_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BUFFER_TOO_SMALL);
+    return 0;
+  }
+
+  if (nonce.size() != EVP_AEAD_AES_GCM_SIV_NONCE_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE);
+    return 0;
+  }
+
+  alignas(16) uint64_t record_auth_key[2];
+  alignas(16) uint64_t record_enc_key[4];
+  aead_aes_gcm_siv_kdf(gcm_siv_ctx->is_128_bit, gcm_siv_ctx, record_auth_key,
+                       record_enc_key, nonce.data());
+
+  alignas(16) uint8_t tag[16] = {0};
+  gcm_siv_asm_polyval(tag, iovecs, aadvecs, (const uint8_t *)record_auth_key,
+                      nonce.data());
+
+  struct aead_aes_gcm_siv_asm_ctx enc_key_expanded;
+
+  if (gcm_siv_ctx->is_128_bit) {
+    aes128gcmsiv_aes_ks_enc_x1(tag, tag, &enc_key_expanded.key[0],
+                               record_enc_key);
+    // Maintain a counter across calls to assembly. The functions internally set
+    // the MSB of the last byte, so we only need to update the counter.
+    alignas(16) uint8_t counter[16];
+    OPENSSL_memcpy(counter, tag, 16);
+    bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/true>(
+        iovecs,
+        [&](const uint8_t *in, uint8_t *out, size_t len) {
+          if (len >= 128) {
+            aes128gcmsiv_enc_msg_x8(in, out, counter, &enc_key_expanded, len);
+          } else {
+            aes128gcmsiv_enc_msg_x4(in, out, counter, &enc_key_expanded, len);
+          }
+          inc_counter(counter, static_cast<uint32_t>(len / AES_BLOCK_SIZE));
+          return true;
+        },
+        [&](const uint8_t *in, uint8_t *out, size_t len) {
+          size_t len_whole = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+          if (len_whole != 0) {
+            if (len_whole >= 128) {
+              aes128gcmsiv_enc_msg_x8(in, out, counter, &enc_key_expanded,
+                                      len_whole);
+            } else {
+              aes128gcmsiv_enc_msg_x4(in, out, counter, &enc_key_expanded,
+                                      len_whole);
+            }
+            in += len_whole;
+            out += len_whole;
+            len -= len_whole;
+          }
+          if (len != 0) {
+            aead_aes_gcm_siv_asm_crypt_last_block(
+                /*is_128_bit=*/true, /*out_last_block=*/out,
+                /*in_last_block=*/in, /*total_in_len=*/in_len, tag,
+                &enc_key_expanded);
+          }
+          return true;
+        });
+  } else {
+    aes256gcmsiv_aes_ks_enc_x1(tag, tag, &enc_key_expanded.key[0],
+                               record_enc_key);
+    alignas(16) uint8_t counter[16];
+    OPENSSL_memcpy(counter, tag, 16);
+    bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/true>(
+        iovecs,
+        [&](const uint8_t *in, uint8_t *out, size_t len) {
+          if (len >= 128) {
+            aes256gcmsiv_enc_msg_x8(in, out, counter, &enc_key_expanded, len);
+          } else {
+            aes256gcmsiv_enc_msg_x4(in, out, counter, &enc_key_expanded, len);
+          }
+          inc_counter(counter, static_cast<uint32_t>(len / AES_BLOCK_SIZE));
+          return true;
+        },
+        [&](const uint8_t *in, uint8_t *out, size_t len) {
+          size_t len_whole = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+          if (len_whole != 0) {
+            if (len_whole >= 128) {
+              aes256gcmsiv_enc_msg_x8(in, out, counter, &enc_key_expanded,
+                                      len_whole);
+            } else {
+              aes256gcmsiv_enc_msg_x4(in, out, counter, &enc_key_expanded,
+                                      len_whole);
+            }
+            in += len_whole;
+            out += len_whole;
+            len -= len_whole;
+          }
+          if (len != 0) {
+            aead_aes_gcm_siv_asm_crypt_last_block(
+                /*is_128_bit=*/false, /*out_last_block=*/out,
+                /*in_last_block=*/in, /*total_in_len=*/in_len, tag,
+                &enc_key_expanded);
+          }
+          return true;
+        });
+  }
+
+  CopyToPrefix(tag, out_tag);
+  *out_tag_len = EVP_AEAD_AES_GCM_SIV_TAG_LEN;
+
+  return 1;
+}
+
+int aead_aes_gcm_siv_asm_openv_detached(const EVP_AEAD_CTX *ctx,
+                                        Span<const CRYPTO_IOVEC> iovecs,
+                                        Span<const uint8_t> nonce,
+                                        Span<const uint8_t> in_tag,
+                                        Span<const CRYPTO_IVEC> aadvecs) {
+  const size_t ad_len = bssl::iovec::TotalLength(aadvecs);
+  const uint64_t ad_len_64 = ad_len;
+  if (ad_len_64 >= (UINT64_C(1) << 61)) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE);
+    return 0;
+  }
+
+  const size_t in_len = bssl::iovec::TotalLength(iovecs);
+  const uint64_t in_len_64 = in_len;
+  if (in_len_64 > UINT64_C(1) << 36 ||
+      in_tag.size() != EVP_AEAD_AES_GCM_SIV_TAG_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_DECRYPT);
+    return 0;
+  }
+
+  if (nonce.size() != EVP_AEAD_AES_GCM_SIV_NONCE_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE);
+    return 0;
+  }
+
+  const struct aead_aes_gcm_siv_asm_ctx *gcm_siv_ctx = asm_ctx_from_ctx(ctx);
+
+  alignas(16) uint64_t record_auth_key[2];
+  alignas(16) uint64_t record_enc_key[4];
+  aead_aes_gcm_siv_kdf(gcm_siv_ctx->is_128_bit, gcm_siv_ctx, record_auth_key,
+                       record_enc_key, nonce.data());
+
+  struct aead_aes_gcm_siv_asm_ctx expanded_key;
+  if (gcm_siv_ctx->is_128_bit) {
+    aes128gcmsiv_aes_ks((const uint8_t *)record_enc_key, &expanded_key.key[0]);
+  } else {
+    aes256gcmsiv_aes_ks((const uint8_t *)record_enc_key, &expanded_key.key[0]);
+  }
+  // calculated_tag is 16*8 bytes, rather than 16 bytes, because
+  // aes[128|256]gcmsiv_dec uses the extra as scratch space.
+  alignas(16) uint8_t calculated_tag[16 * 8] = {0};
+
+  OPENSSL_memset(calculated_tag, 0, EVP_AEAD_AES_GCM_SIV_TAG_LEN);
+  bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/false>(
+      aadvecs,
+      [&](const uint8_t *in, size_t len) {
+        aesgcmsiv_polyval_horner(calculated_tag,
+                                 (const uint8_t *)record_auth_key, in,
+                                 len / AES_BLOCK_SIZE);
+        return true;
+      },
+      [&](const uint8_t *in, size_t len) {
+        size_t len_whole = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+        if (len_whole != 0) {
+          aesgcmsiv_polyval_horner(calculated_tag,
+                                   (const uint8_t *)record_auth_key, in,
+                                   len_whole / AES_BLOCK_SIZE);
+          in += len_whole;
+          len -= len_whole;
+        }
+        if (len != 0) {
+          uint8_t pad_buf[AES_BLOCK_SIZE];
+          OPENSSL_memcpy(pad_buf, in, len);
+          OPENSSL_memset(pad_buf + len, 0, AES_BLOCK_SIZE - len);
+          aesgcmsiv_polyval_horner(
+              calculated_tag, (const uint8_t *)record_auth_key, pad_buf, 1);
+        }
+        return true;
+      });
+
+  alignas(16) uint8_t htable[16 * 6];
+  aesgcmsiv_htable6_init(htable, (const uint8_t *)record_auth_key);
+
+  size_t blocks = 0;
+  bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/true>(
+      iovecs,
+      [&](const uint8_t *in, uint8_t *out, size_t len) {
+        // aes[128|256]gcmsiv_dec needs access to the claimed tag. So it's put
+        // into its scratch space. The function may clobber the claimed tag, so
+        // this is copied before each call.
+        OPENSSL_memcpy(calculated_tag + 16, in_tag.data(),
+                       EVP_AEAD_AES_GCM_SIV_TAG_LEN);
+        inc_counter(calculated_tag + 16, static_cast<uint32_t>(blocks));
+        if (gcm_siv_ctx->is_128_bit) {
+          aes128gcmsiv_dec(in, out, calculated_tag, htable, &expanded_key, len);
+        } else {
+          aes256gcmsiv_dec(in, out, calculated_tag, htable, &expanded_key, len);
+        }
+        blocks += len / AES_BLOCK_SIZE;
+        return true;
+      },
+      [&](const uint8_t *in, uint8_t *out, size_t len) {
+        size_t len_whole = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+        if (len_whole != 0) {
+          // aes[128|256]gcmsiv_dec needs access to the claimed tag. So it's put
+          // into its scratch space. The function may clobber the claimed tag,
+          // so this is copied before each call.
+          OPENSSL_memcpy(calculated_tag + 16, in_tag.data(),
+                         EVP_AEAD_AES_GCM_SIV_TAG_LEN);
+          inc_counter(calculated_tag + 16, static_cast<uint32_t>(blocks));
+          if (gcm_siv_ctx->is_128_bit) {
+            aes128gcmsiv_dec(in, out, calculated_tag, htable, &expanded_key,
+                             len_whole);
+          } else {
+            aes256gcmsiv_dec(in, out, calculated_tag, htable, &expanded_key,
+                             len_whole);
+          }
+          in += len_whole;
+          out += len_whole;
+          len -= len_whole;
+        }
+        if (len != 0) {
+          aead_aes_gcm_siv_asm_crypt_last_block(
+              gcm_siv_ctx->is_128_bit, /*out_last_block=*/out,
+              /*in_last_block=*/in, /*total_in_len=*/in_len, in_tag.data(),
+              &expanded_key);
+          uint8_t pad_buf[AES_BLOCK_SIZE];
+          OPENSSL_memcpy(pad_buf, out, len);
+          OPENSSL_memset(pad_buf + len, 0, AES_BLOCK_SIZE - len);
+          aesgcmsiv_polyval_horner(
+              calculated_tag, (const uint8_t *)record_auth_key, pad_buf, 1);
+        }
+        return true;
+      });
+
+  uint8_t length_block[16];
+  CRYPTO_store_u64_le(length_block, ad_len * 8);
+  CRYPTO_store_u64_le(length_block + 8, in_len * 8);
+  aesgcmsiv_polyval_horner(calculated_tag, (const uint8_t *)record_auth_key,
+                           length_block, 1);
+
+  for (size_t i = 0; i < 12; i++) {
+    calculated_tag[i] ^= nonce[i];
+  }
+
+  calculated_tag[15] &= 0x7f;
+
+  if (gcm_siv_ctx->is_128_bit) {
+    aes128gcmsiv_ecb_enc_block(calculated_tag, calculated_tag, &expanded_key);
+  } else {
+    aes256gcmsiv_ecb_enc_block(calculated_tag, calculated_tag, &expanded_key);
+  }
+
+  if (CRYPTO_memcmp(calculated_tag, in_tag.data(),
+                    EVP_AEAD_AES_GCM_SIV_TAG_LEN) != 0) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_DECRYPT);
+    return 0;
+  }
+
+  return 1;
+}
+
+const EVP_AEAD aead_aes_128_gcm_siv_asm = {
+    16,                              // key length
+    EVP_AEAD_AES_GCM_SIV_NONCE_LEN,  // nonce length
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // overhead
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // max tag length
+
+    aead_aes_gcm_siv_asm_init,
+    nullptr /* init_with_direction */,
+    aead_aes_gcm_siv_asm_cleanup,
+    nullptr /* openv */,
+    aead_aes_gcm_siv_asm_sealv,
+    aead_aes_gcm_siv_asm_openv_detached,
+    nullptr /* get_iv */,
+    nullptr /* tag_len */,
+};
+
+const EVP_AEAD aead_aes_256_gcm_siv_asm = {
+    32,                              // key length
+    EVP_AEAD_AES_GCM_SIV_NONCE_LEN,  // nonce length
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // overhead
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // max tag length
+
+    aead_aes_gcm_siv_asm_init,
+    nullptr /* init_with_direction */,
+    aead_aes_gcm_siv_asm_cleanup,
+    nullptr /* openv */,
+    aead_aes_gcm_siv_asm_sealv,
+    aead_aes_gcm_siv_asm_openv_detached,
+    nullptr /* get_iv */,
+    nullptr /* tag_len */,
+};
+
+#endif  // AES_GCM_SIV_ASM
+
+
+struct aead_aes_gcm_siv_ctx {
+  union {
+    double align;
+    AES_KEY ks;
+  } ks;
+  block128_f kgk_block;
+  unsigned is_256 : 1;
+};
+
+static_assert(sizeof(((EVP_AEAD_CTX *)nullptr)->state) >=
+                  sizeof(struct aead_aes_gcm_siv_ctx),
+              "AEAD state is too small");
+static_assert(alignof(union evp_aead_ctx_st_state) >=
+                  alignof(struct aead_aes_gcm_siv_ctx),
+              "AEAD state has insufficient alignment");
+
+int aead_aes_gcm_siv_init(EVP_AEAD_CTX *ctx, const uint8_t *key, size_t key_len,
+                          size_t tag_len) {
+  const size_t key_bits = key_len * 8;
+
+  if (key_bits != 128 && key_bits != 256) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_KEY_LENGTH);
+    return 0;  // EVP_AEAD_CTX_init should catch this.
+  }
+
+  if (tag_len == EVP_AEAD_DEFAULT_TAG_LENGTH) {
+    tag_len = EVP_AEAD_AES_GCM_SIV_TAG_LEN;
+  }
+  if (tag_len != EVP_AEAD_AES_GCM_SIV_TAG_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TAG_TOO_LARGE);
+    return 0;
+  }
+
+  struct aead_aes_gcm_siv_ctx *gcm_siv_ctx =
+      (struct aead_aes_gcm_siv_ctx *)&ctx->state;
+  OPENSSL_memset(gcm_siv_ctx, 0, sizeof(struct aead_aes_gcm_siv_ctx));
+
+  aes_ctr_set_key(&gcm_siv_ctx->ks.ks, nullptr, &gcm_siv_ctx->kgk_block, key,
+                  key_len);
+  gcm_siv_ctx->is_256 = (key_len == 32);
+  ctx->tag_len = tag_len;
+
+  return 1;
+}
+
+void aead_aes_gcm_siv_cleanup(EVP_AEAD_CTX *ctx) {}
+
+// gcm_siv_crypt encrypts (or decrypts—it's the same thing) bytes from `in` to
+// `out` in the `iovec`, using the block function `enc_block` with `key` in
+// counter mode, starting at `initial_counter`. This differs from the
+// traditional counter mode code in that the counter is handled little-endian,
+// only the first four bytes are used and the GCM-SIV tweak to the final byte
+// is applied. The `in` and `out` pointers may be equal but otherwise must not
+// alias.
+void gcm_siv_crypt(Span<const CRYPTO_IOVEC> iovecs,
+                   const uint8_t initial_counter[AES_BLOCK_SIZE],
+                   block128_f enc_block, const AES_KEY *key) {
+  uint8_t counter[16];
+
+  OPENSSL_memcpy(counter, initial_counter, AES_BLOCK_SIZE);
+  counter[15] |= 0x80;
+
+  auto crypt_bytes = [&](const uint8_t *in, uint8_t *out, size_t len) {
+    for (size_t done = 0; done < len;) {
+      uint8_t keystream[AES_BLOCK_SIZE];
+      enc_block(counter, keystream, key);
+      inc_counter(counter, 1);
+
+      size_t todo = AES_BLOCK_SIZE;
+      if (len - done < todo) {
+        todo = len - done;
+      }
+
+      for (size_t i = 0; i < todo; i++) {
+        out[done + i] = keystream[i] ^ in[done + i];
+      }
+
+      done += todo;
+    }
+    return true;
+  };
+
+  bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/true>(
+      iovecs, crypt_bytes, crypt_bytes);
+}
+
+
+// POLYVAL.
+//
+// POLYVAL is a polynomial authenticator that operates over a field very
+// similar to the one that GHASH uses. See
+// https://www.rfc-editor.org/rfc/rfc8452.html#section-3.
+
+// POLYVAL(H, X₁, ..., Xₙ) =
+// ByteReverse(GHASH(mulX_GHASH(ByteReverse(H)), ByteReverse(X₁), ...,
+// ByteReverse(Xₙ))).
+//
+// See https://www.rfc-editor.org/rfc/rfc8452.html#appendix-A.
+
+struct polyval_ctx {
+  uint8_t S[16];
+  u128 Htable[16];
+  gmult_func gmult;
+  ghash_func ghash;
+};
+
+// byte_reverse reverses the order of the bytes in `b->c`.
+void byte_reverse(uint8_t b[16]) {
+  uint64_t hi = CRYPTO_load_u64_le(b);
+  uint64_t lo = CRYPTO_load_u64_le(b + 8);
+  CRYPTO_store_u64_le(b, CRYPTO_bswap8(lo));
+  CRYPTO_store_u64_le(b + 8, CRYPTO_bswap8(hi));
+}
+
+// reverse_and_mulX_ghash interprets `b` as a reversed element of the GHASH
+// field, multiplies that by 'x' and serialises the result back into `b`, but
+// with GHASH's backwards bit ordering.
+void reverse_and_mulX_ghash(uint8_t b[16]) {
+  uint64_t hi = CRYPTO_load_u64_le(b);
+  uint64_t lo = CRYPTO_load_u64_le(b + 8);
+  const crypto_word_t carry = constant_time_eq_w(hi & 1, 1);
+  hi >>= 1;
+  hi |= lo << 63;
+  lo >>= 1;
+  lo ^= ((uint64_t)constant_time_select_w(carry, 0xe1, 0)) << 56;
+
+  CRYPTO_store_u64_le(b, CRYPTO_bswap8(lo));
+  CRYPTO_store_u64_le(b + 8, CRYPTO_bswap8(hi));
+}
+
+void crypto_polyval_init(struct polyval_ctx *ctx, const uint8_t key[16]) {
+  alignas(8) uint8_t H[16];
+  OPENSSL_memcpy(H, key, 16);
+  reverse_and_mulX_ghash(H);
+
+  CRYPTO_ghash_init(&ctx->gmult, &ctx->ghash, ctx->Htable, H);
+  OPENSSL_memset(&ctx->S, 0, sizeof(ctx->S));
+}
+
+void crypto_polyval_update_blocks(struct polyval_ctx *ctx, const uint8_t *in,
+                                  size_t in_len) {
+  assert((in_len & 15) == 0);
+  alignas(8) uint8_t buf[32 * 16];
+
+  while (in_len > 0) {
+    size_t todo = in_len;
+    if (todo > sizeof(buf)) {
+      todo = sizeof(buf);
+    }
+    OPENSSL_memcpy(buf, in, todo);
+    in += todo;
+    in_len -= todo;
+
+    size_t blocks = todo / 16;
+    for (size_t i = 0; i < blocks; i++) {
+      byte_reverse(buf + 16 * i);
+    }
+
+    ctx->ghash(ctx->S, ctx->Htable, buf, todo);
+  }
+}
+
+void crypto_polyval_finish(const struct polyval_ctx *ctx, uint8_t out[16]) {
+  OPENSSL_memcpy(out, &ctx->S, 16);
+  byte_reverse(out);
+}
+
+// gcm_siv_polyval evaluates POLYVAL at `auth_key` on the given plaintext and
+// AD. The result is written to `out_tag`.
+void gcm_siv_polyval(uint8_t out_tag[16], Span<const CRYPTO_IOVEC> iovecs,
+                     bool encrypt, Span<const CRYPTO_IVEC> aadvecs,
+                     const uint8_t auth_key[16],
+                     const uint8_t nonce[EVP_AEAD_AES_GCM_SIV_NONCE_LEN]) {
+  struct polyval_ctx polyval_ctx;
+  crypto_polyval_init(&polyval_ctx, auth_key);
+
+  auto f_whole = [&](const uint8_t *in, size_t len) {
+    crypto_polyval_update_blocks(&polyval_ctx, in, len);
+    return true;
+  };
+  auto f_final = [&](const uint8_t *in, size_t len) {
+    size_t len_whole = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+    if (len_whole != 0) {
+      crypto_polyval_update_blocks(&polyval_ctx, in, len_whole);
+      in += len_whole;
+      len -= len_whole;
+    }
+    if (len != 0) {
+      uint8_t pad_buf[AES_BLOCK_SIZE];
+      OPENSSL_memcpy(pad_buf, in, len);
+      OPENSSL_memset(pad_buf + len, 0, AES_BLOCK_SIZE - len);
+      crypto_polyval_update_blocks(&polyval_ctx, pad_buf, AES_BLOCK_SIZE);
+    }
+    return true;
+  };
+
+  bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/false>(
+      aadvecs, f_whole, f_final);
+
+  if (encrypt) {
+    bssl::iovec::ForEachBlockRange<AES_BLOCK_SIZE, /*WriteOut=*/false>(
+        iovecs, f_whole, f_final);
+  } else {
+    bssl::iovec::ForEachOutBlockRange<AES_BLOCK_SIZE>(iovecs, f_whole, f_final);
+  }
+
+  uint8_t length_block[16];
+  CRYPTO_store_u64_le(length_block,
+                      ((uint64_t)bssl::iovec::TotalLength(aadvecs)) * 8);
+  CRYPTO_store_u64_le(length_block + 8,
+                      ((uint64_t)bssl::iovec::TotalLength(iovecs)) * 8);
+  crypto_polyval_update_blocks(&polyval_ctx, length_block,
+                               sizeof(length_block));
+
+  crypto_polyval_finish(&polyval_ctx, out_tag);
+  for (size_t i = 0; i < EVP_AEAD_AES_GCM_SIV_NONCE_LEN; i++) {
+    out_tag[i] ^= nonce[i];
+  }
+  out_tag[15] &= 0x7f;
+}
+
+// gcm_siv_record_keys contains the keys used for a specific GCM-SIV record.
+struct gcm_siv_record_keys {
+  uint8_t auth_key[16];
+  union {
+    double align;
+    AES_KEY ks;
+  } enc_key;
+  block128_f enc_block;
+};
+
+// gcm_siv_keys calculates the keys for a specific GCM-SIV record with the
+// given nonce and writes them to `*out_keys`.
+void gcm_siv_keys(const struct aead_aes_gcm_siv_ctx *gcm_siv_ctx,
+                  struct gcm_siv_record_keys *out_keys,
+                  const uint8_t nonce[EVP_AEAD_AES_GCM_SIV_NONCE_LEN]) {
+  const AES_KEY *const key = &gcm_siv_ctx->ks.ks;
+  uint8_t key_material[(128 /* POLYVAL key */ + 256 /* max AES key */) / 8];
+  const size_t blocks_needed = gcm_siv_ctx->is_256 ? 6 : 4;
+
+  uint8_t counter[AES_BLOCK_SIZE];
+  OPENSSL_memset(counter, 0, AES_BLOCK_SIZE - EVP_AEAD_AES_GCM_SIV_NONCE_LEN);
+  OPENSSL_memcpy(counter + AES_BLOCK_SIZE - EVP_AEAD_AES_GCM_SIV_NONCE_LEN,
+                 nonce, EVP_AEAD_AES_GCM_SIV_NONCE_LEN);
+  for (size_t i = 0; i < blocks_needed; i++) {
+    counter[0] = i;
+
+    uint8_t ciphertext[AES_BLOCK_SIZE];
+    gcm_siv_ctx->kgk_block(counter, ciphertext, key);
+    OPENSSL_memcpy(&key_material[i * 8], ciphertext, 8);
+  }
+
+  OPENSSL_memcpy(out_keys->auth_key, key_material, 16);
+  // Note the `ctr128_f` function uses a big-endian counter, while AES-GCM-SIV
+  // uses a little-endian counter. We ignore the return value and only use
+  // `block128_f`. This has a significant performance cost for the fallback
+  // bitsliced AES implementations (bsaes and aes_nohw).
+  //
+  // We currently do not consider AES-GCM-SIV to be performance-sensitive on
+  // client hardware. If this changes, we can write little-endian `ctr128_f`
+  // functions.
+  aes_ctr_set_key(&out_keys->enc_key.ks, nullptr, &out_keys->enc_block,
+                  key_material + 16, gcm_siv_ctx->is_256 ? 32 : 16);
+}
+
+int aead_aes_gcm_siv_sealv(const EVP_AEAD_CTX *ctx,
+                           Span<const CRYPTO_IOVEC> iovecs,
+                           Span<uint8_t> out_tag, size_t *out_tag_len,
+                           Span<const uint8_t> nonce,
+                           Span<const CRYPTO_IVEC> aadvecs) {
+  const struct aead_aes_gcm_siv_ctx *gcm_siv_ctx =
+      (struct aead_aes_gcm_siv_ctx *)&ctx->state;
+  size_t in_len = bssl::iovec::TotalLength(iovecs);
+  size_t ad_len = bssl::iovec::TotalLength(aadvecs);
+  const uint64_t in_len_64 = in_len;
+  const uint64_t ad_len_64 = ad_len;
+
+  if (in_len + EVP_AEAD_AES_GCM_SIV_TAG_LEN < in_len ||
+      in_len_64 > (UINT64_C(1) << 36) || ad_len_64 >= (UINT64_C(1) << 61)) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE);
+    return 0;
+  }
+
+  if (out_tag.size() < EVP_AEAD_AES_GCM_SIV_TAG_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BUFFER_TOO_SMALL);
+    return 0;
+  }
+
+  if (nonce.size() != EVP_AEAD_AES_GCM_SIV_NONCE_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE);
+    return 0;
+  }
+
+  struct gcm_siv_record_keys keys;
+  gcm_siv_keys(gcm_siv_ctx, &keys, nonce.data());
+
+  uint8_t tag[EVP_AEAD_AES_GCM_SIV_TAG_LEN];
+  gcm_siv_polyval(tag, iovecs, true, aadvecs, keys.auth_key, nonce.data());
+  keys.enc_block(tag, tag, &keys.enc_key.ks);
+
+  gcm_siv_crypt(iovecs, tag, keys.enc_block, &keys.enc_key.ks);
+
+  CopyToPrefix(tag, out_tag);
+  *out_tag_len = EVP_AEAD_AES_GCM_SIV_TAG_LEN;
+
+  return 1;
+}
+
+int aead_aes_gcm_siv_openv_detached(const EVP_AEAD_CTX *ctx,
+                                    Span<const CRYPTO_IOVEC> iovecs,
+                                    Span<const uint8_t> nonce,
+                                    Span<const uint8_t> in_tag,
+                                    Span<const CRYPTO_IVEC> aadvecs) {
+  const uint64_t ad_len_64 = bssl::iovec::TotalLength(aadvecs);
+  if (ad_len_64 >= (UINT64_C(1) << 61)) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE);
+    return 0;
+  }
+
+  const uint64_t in_len_64 = bssl::iovec::TotalLength(iovecs);
+  if (in_tag.size() != EVP_AEAD_AES_GCM_SIV_TAG_LEN ||
+      in_len_64 > (UINT64_C(1) << 36)) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_DECRYPT);
+    return 0;
+  }
+
+  if (nonce.size() != EVP_AEAD_AES_GCM_SIV_NONCE_LEN) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE);
+    return 0;
+  }
+
+  const struct aead_aes_gcm_siv_ctx *gcm_siv_ctx =
+      (struct aead_aes_gcm_siv_ctx *)&ctx->state;
+
+  struct gcm_siv_record_keys keys;
+  gcm_siv_keys(gcm_siv_ctx, &keys, nonce.data());
+
+  gcm_siv_crypt(iovecs, in_tag.data(), keys.enc_block, &keys.enc_key.ks);
+
+  uint8_t expected_tag[EVP_AEAD_AES_GCM_SIV_TAG_LEN];
+  gcm_siv_polyval(expected_tag, iovecs, false, aadvecs, keys.auth_key,
+                  nonce.data());
+  keys.enc_block(expected_tag, expected_tag, &keys.enc_key.ks);
+
+  if (CRYPTO_memcmp(expected_tag, in_tag.data(), sizeof(expected_tag)) != 0) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_DECRYPT);
+    return 0;
+  }
+
+  return 1;
+}
+
+const EVP_AEAD aead_aes_128_gcm_siv = {
+    16,                              // key length
+    EVP_AEAD_AES_GCM_SIV_NONCE_LEN,  // nonce length
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // overhead
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // max tag length
+
+    aead_aes_gcm_siv_init,
+    nullptr /* init_with_direction */,
+    aead_aes_gcm_siv_cleanup,
+    nullptr /* openv */,
+    aead_aes_gcm_siv_sealv,
+    aead_aes_gcm_siv_openv_detached,
+    nullptr /* get_iv */,
+    nullptr /* tag_len */,
+};
+
+const EVP_AEAD aead_aes_256_gcm_siv = {
+    32,                              // key length
+    EVP_AEAD_AES_GCM_SIV_NONCE_LEN,  // nonce length
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // overhead
+    EVP_AEAD_AES_GCM_SIV_TAG_LEN,    // max tag length
+
+    aead_aes_gcm_siv_init,
+    nullptr /* init_with_direction */,
+    aead_aes_gcm_siv_cleanup,
+    nullptr /* openv */,
+    aead_aes_gcm_siv_sealv,
+    aead_aes_gcm_siv_openv_detached,
+    nullptr /* get_iv */,
+    nullptr /* tag_len */,
+};
+}  // namespace
+
+const EVP_AEAD *EVP_aead_aes_128_gcm_siv() {
+#if defined(AES_GCM_SIV_ASM)
+  if (aes_gcm_siv_asm_capable()) {
+    return &aead_aes_128_gcm_siv_asm;
+  }
+#endif
+  return &aead_aes_128_gcm_siv;
+}
+
+const EVP_AEAD *EVP_aead_aes_256_gcm_siv() {
+#if defined(AES_GCM_SIV_ASM)
+  if (aes_gcm_siv_asm_capable()) {
+    return &aead_aes_256_gcm_siv_asm;
+  }
+#endif
+  return &aead_aes_256_gcm_siv;
+}