re2 1.24.0 → 1.25.0

package/vendor/abseil-cpp/absl/hash/internal/city.cc

@@ -21,13 +21,13 @@
 
 #include "absl/hash/internal/city.h"
 
-#include <string.h>  // for memcpy and memset
 #include <algorithm>
 
 #include "absl/base/config.h"
 #include "absl/base/internal/endian.h"
 #include "absl/base/internal/unaligned_access.h"
 #include "absl/base/optimization.h"
+#include "absl/numeric/bits.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -68,11 +68,6 @@ static uint32_t fmix(uint32_t h) {
   return h;
 }
 
-static uint32_t Rotate32(uint32_t val, int shift) {
-  // Avoid shifting by 32: doing so yields an undefined result.
-  return shift == 0 ? val : ((val >> shift) | (val << (32 - shift)));
-}
-
 #undef PERMUTE3
 #define PERMUTE3(a, b, c) \
   do {                    \
@@ -83,10 +78,10 @@ static uint32_t Rotate32(uint32_t val, int shift) {
 static uint32_t Mur(uint32_t a, uint32_t h) {
   // Helper from Murmur3 for combining two 32-bit values.
   a *= c1;
-  a = Rotate32(a, 17);
+  a = absl::rotr(a, 17);
   a *= c2;
   h ^= a;
-  h = Rotate32(h, 19);
+  h = absl::rotr(h, 19);
   return h * 5 + 0xe6546b64;
 }
 
@@ -131,44 +126,44 @@ uint32_t CityHash32(const char *s, size_t len) {
   // len > 24
   uint32_t h = static_cast<uint32_t>(len), g = c1 * h, f = g;
 
-  uint32_t a0 = Rotate32(Fetch32(s + len - 4) * c1, 17) * c2;
-  uint32_t a1 = Rotate32(Fetch32(s + len - 8) * c1, 17) * c2;
-  uint32_t a2 = Rotate32(Fetch32(s + len - 16) * c1, 17) * c2;
-  uint32_t a3 = Rotate32(Fetch32(s + len - 12) * c1, 17) * c2;
-  uint32_t a4 = Rotate32(Fetch32(s + len - 20) * c1, 17) * c2;
+  uint32_t a0 = absl::rotr(Fetch32(s + len - 4) * c1, 17) * c2;
+  uint32_t a1 = absl::rotr(Fetch32(s + len - 8) * c1, 17) * c2;
+  uint32_t a2 = absl::rotr(Fetch32(s + len - 16) * c1, 17) * c2;
+  uint32_t a3 = absl::rotr(Fetch32(s + len - 12) * c1, 17) * c2;
+  uint32_t a4 = absl::rotr(Fetch32(s + len - 20) * c1, 17) * c2;
   h ^= a0;
-  h = Rotate32(h, 19);
+  h = absl::rotr(h, 19);
   h = h * 5 + 0xe6546b64;
   h ^= a2;
-  h = Rotate32(h, 19);
+  h = absl::rotr(h, 19);
   h = h * 5 + 0xe6546b64;
   g ^= a1;
-  g = Rotate32(g, 19);
+  g = absl::rotr(g, 19);
   g = g * 5 + 0xe6546b64;
   g ^= a3;
-  g = Rotate32(g, 19);
+  g = absl::rotr(g, 19);
   g = g * 5 + 0xe6546b64;
   f += a4;
-  f = Rotate32(f, 19);
+  f = absl::rotr(f, 19);
   f = f * 5 + 0xe6546b64;
   size_t iters = (len - 1) / 20;
   do {
-    uint32_t b0 = Rotate32(Fetch32(s) * c1, 17) * c2;
+    uint32_t b0 = absl::rotr(Fetch32(s) * c1, 17) * c2;
     uint32_t b1 = Fetch32(s + 4);
-    uint32_t b2 = Rotate32(Fetch32(s + 8) * c1, 17) * c2;
-    uint32_t b3 = Rotate32(Fetch32(s + 12) * c1, 17) * c2;
+    uint32_t b2 = absl::rotr(Fetch32(s + 8) * c1, 17) * c2;
+    uint32_t b3 = absl::rotr(Fetch32(s + 12) * c1, 17) * c2;
     uint32_t b4 = Fetch32(s + 16);
     h ^= b0;
-    h = Rotate32(h, 18);
+    h = absl::rotr(h, 18);
     h = h * 5 + 0xe6546b64;
     f += b1;
-    f = Rotate32(f, 19);
+    f = absl::rotr(f, 19);
     f = f * c1;
     g += b2;
-    g = Rotate32(g, 18);
+    g = absl::rotr(g, 18);
     g = g * 5 + 0xe6546b64;
     h ^= b3 + b1;
-    h = Rotate32(h, 19);
+    h = absl::rotr(h, 19);
     h = h * 5 + 0xe6546b64;
     g ^= b4;
     g = absl::gbswap_32(g) * 5;
@@ -178,26 +173,19 @@ uint32_t CityHash32(const char *s, size_t len) {
     PERMUTE3(f, h, g);
     s += 20;
   } while (--iters != 0);
-  g = Rotate32(g, 11) * c1;
-  g = Rotate32(g, 17) * c1;
-  f = Rotate32(f, 11) * c1;
-  f = Rotate32(f, 17) * c1;
-  h = Rotate32(h + g, 19);
+  g = absl::rotr(g, 11) * c1;
+  g = absl::rotr(g, 17) * c1;
+  f = absl::rotr(f, 11) * c1;
+  f = absl::rotr(f, 17) * c1;
+  h = absl::rotr(h + g, 19);
   h = h * 5 + 0xe6546b64;
-  h = Rotate32(h, 17) * c1;
-  h = Rotate32(h + f, 19);
+  h = absl::rotr(h, 17) * c1;
+  h = absl::rotr(h + f, 19);
   h = h * 5 + 0xe6546b64;
-  h = Rotate32(h, 17) * c1;
+  h = absl::rotr(h, 17) * c1;
   return h;
 }
 
-// Bitwise right rotate.  Normally this will compile to a single
-// instruction, especially if the shift is a manifest constant.
-static uint64_t Rotate(uint64_t val, int shift) {
-  // Avoid shifting by 64: doing so yields an undefined result.
-  return shift == 0 ? val : ((val >> shift) | (val << (64 - shift)));
-}
-
 static uint64_t ShiftMix(uint64_t val) { return val ^ (val >> 47); }
 
 static uint64_t HashLen16(uint64_t u, uint64_t v, uint64_t mul) {
@@ -220,8 +208,8 @@ static uint64_t HashLen0to16(const char *s, size_t len) {
     uint64_t mul = k2 + len * 2;
     uint64_t a = Fetch64(s) + k2;
     uint64_t b = Fetch64(s + len - 8);
-    uint64_t c = Rotate(b, 37) * mul + a;
-    uint64_t d = (Rotate(a, 25) + b) * mul;
+    uint64_t c = absl::rotr(b, 37) * mul + a;
+    uint64_t d = (absl::rotr(a, 25) + b) * mul;
     return HashLen16(c, d, mul);
   }
   if (len >= 4) {
@@ -248,8 +236,8 @@ static uint64_t HashLen17to32(const char *s, size_t len) {
   uint64_t b = Fetch64(s + 8);
   uint64_t c = Fetch64(s + len - 8) * mul;
   uint64_t d = Fetch64(s + len - 16) * k2;
-  return HashLen16(Rotate(a + b, 43) + Rotate(c, 30) + d,
-                   a + Rotate(b + k2, 18) + c, mul);
+  return HashLen16(absl::rotr(a + b, 43) + absl::rotr(c, 30) + d,
+                   a + absl::rotr(b + k2, 18) + c, mul);
 }
 
 // Return a 16-byte hash for 48 bytes.  Quick and dirty.
@@ -257,11 +245,11 @@ static uint64_t HashLen17to32(const char *s, size_t len) {
 static std::pair<uint64_t, uint64_t> WeakHashLen32WithSeeds(
     uint64_t w, uint64_t x, uint64_t y, uint64_t z, uint64_t a, uint64_t b) {
   a += w;
-  b = Rotate(b + a + z, 21);
+  b = absl::rotr(b + a + z, 21);
   uint64_t c = a;
   a += x;
   a += y;
-  b += Rotate(a, 44);
+  b += absl::rotr(a, 44);
   return std::make_pair(a + z, b + c);
 }
 
@@ -284,10 +272,10 @@ static uint64_t HashLen33to64(const char *s, size_t len) {
   uint64_t f = Fetch64(s + 24) * 9;
   uint64_t g = Fetch64(s + len - 8);
   uint64_t h = Fetch64(s + len - 16) * mul;
-  uint64_t u = Rotate(a + g, 43) + (Rotate(b, 30) + c) * 9;
+  uint64_t u = absl::rotr(a + g, 43) + (absl::rotr(b, 30) + c) * 9;
   uint64_t v = ((a + g) ^ d) + f + 1;
   uint64_t w = absl::gbswap_64((u + v) * mul) + h;
-  uint64_t x = Rotate(e + f, 42) + c;
+  uint64_t x = absl::rotr(e + f, 42) + c;
   uint64_t y = (absl::gbswap_64((v + w) * mul) + g) * mul;
   uint64_t z = e + f + c;
   a = absl::gbswap_64((x + z) * mul + y) + b;
@@ -320,11 +308,11 @@ uint64_t CityHash64(const char *s, size_t len) {
   // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
   len = (len - 1) & ~static_cast<size_t>(63);
   do {
-    x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
-    y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
+    x = absl::rotr(x + y + v.first + Fetch64(s + 8), 37) * k1;
+    y = absl::rotr(y + v.second + Fetch64(s + 48), 42) * k1;
     x ^= w.second;
     y += v.first + Fetch64(s + 40);
-    z = Rotate(z + w.first, 33) * k1;
+    z = absl::rotr(z + w.first, 33) * k1;
     v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
     w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
     std::swap(z, x);

package/vendor/abseil-cpp/absl/hash/internal/hash.cc

@@ -39,6 +39,13 @@
 #include <xmmintrin.h>
 #endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
 
+#ifdef ABSL_AES_INTERNAL_HAVE_ARM_SIMD
+#error ABSL_AES_INTERNAL_HAVE_ARM_SIMD cannot be directly set
+#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(__ARM_FEATURE_CRYPTO)
+#include <arm_neon.h>
+#define ABSL_AES_INTERNAL_HAVE_ARM_SIMD
+#endif  // ABSL_INTERNAL_HAVE_ARM_NEON
+
 namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace hash_internal {
@@ -49,39 +56,159 @@ void PrefetchFutureDataToLocalCache(const uint8_t* ptr) {
   PrefetchToLocalCache(ptr + 5 * ABSL_CACHELINE_SIZE);
 }
 
-#ifdef ABSL_AES_INTERNAL_HAVE_X86_SIMD
-uint64_t Mix4x16Vectors(__m128i a, __m128i b, __m128i c, __m128i d) {
-  // res128 = encrypt(a + c, d) + decrypt(b - d, a)
-  auto res128 = _mm_add_epi64(_mm_aesenc_si128(_mm_add_epi64(a, c), d),
-                              _mm_aesdec_si128(_mm_sub_epi64(b, d), a));
-  auto x64 = static_cast<uint64_t>(_mm_cvtsi128_si64(res128));
-  auto y64 = static_cast<uint64_t>(_mm_extract_epi64(res128, 1));
+#if defined(ABSL_AES_INTERNAL_HAVE_X86_SIMD) || \
+    defined(ABSL_AES_INTERNAL_HAVE_ARM_SIMD)
+
+#if defined(ABSL_AES_INTERNAL_HAVE_X86_SIMD)
+using Vector128 = __m128i;
+
+inline Vector128 Load128(const uint8_t* ptr) {
+  return _mm_loadu_si128(reinterpret_cast<const Vector128*>(ptr));
+}
+
+inline Vector128 Set128(uint64_t a, uint64_t b) {
+  return _mm_set_epi64x(static_cast<int64_t>(a), static_cast<int64_t>(b));
+}
+
+inline Vector128 Add128(Vector128 a, Vector128 b) {
+  return _mm_add_epi64(a, b);
+}
+
+inline Vector128 Sub128(Vector128 a, Vector128 b) {
+  return _mm_sub_epi64(a, b);
+}
+
+// Bits of the second argument to Encrypt128/Decrypt128 are XORed with the
+// first argument after encryption/decryption.
+
+inline Vector128 Encrypt128(Vector128 data, Vector128 key) {
+  return _mm_aesenc_si128(data, key);
+}
+
+inline Vector128 Decrypt128(Vector128 data, Vector128 key) {
+  return _mm_aesdec_si128(data, key);
+}
+
+// We use each value as the first argument to shuffle all the bits around. We do
+// not add any salt to the state or loaded data, instead we vary instructions
+// used to mix bits Encrypt128/Decrypt128 and Add128/Sub128. On x86,
+// Add128/Sub128 are combined to one instruction with data loading like
+// `vpaddq xmm1, xmm0, xmmword ptr [rdi]`.
+
+inline Vector128 MixA(Vector128 a, Vector128 state) {
+  return Decrypt128(Add128(state, a), state);
+}
+
+inline Vector128 MixB(Vector128 b, Vector128 state) {
+  return Decrypt128(Sub128(state, b), state);
+}
+
+inline Vector128 MixC(Vector128 c, Vector128 state) {
+  return Encrypt128(Add128(state, c), state);
+}
+
+inline Vector128 MixD(Vector128 d, Vector128 state) {
+  return Encrypt128(Sub128(state, d), state);
+}
+
+inline uint64_t ExtractLow64(Vector128 v) {
+  return static_cast<uint64_t>(_mm_cvtsi128_si64(v));
+}
+
+inline uint64_t ExtractHigh64(Vector128 v) {
+  return static_cast<uint64_t>(_mm_extract_epi64(v, 1));
+}
+
+inline uint64_t Mix4x16Vectors(Vector128 a, Vector128 b, Vector128 c,
+                               Vector128 d) {
+  Vector128 res128 =
+      Add128(Encrypt128(Add128(a, c), d), Decrypt128(Sub128(b, d), a));
+  uint64_t x64 = ExtractLow64(res128);
+  uint64_t y64 = ExtractHigh64(res128);
   return x64 ^ y64;
 }
 
+#else  // ABSL_AES_INTERNAL_HAVE_ARM_SIMD
+
+using Vector128 = uint8x16_t;
+
+inline Vector128 Load128(const uint8_t* ptr) { return vld1q_u8(ptr); }
+
+inline Vector128 Set128(uint64_t a, uint64_t b) {
+  return vreinterpretq_u8_u64(vsetq_lane_u64(a, vdupq_n_u64(b), 1));
+}
+
+inline Vector128 Add128(Vector128 a, Vector128 b) {
+  return vreinterpretq_u8_u64(
+      vaddq_u64(vreinterpretq_u64_u8(a), vreinterpretq_u64_u8(b)));
+}
+
+// Bits of the second argument to Decrypt128/Encrypt128 are XORed with the
+// state argument BEFORE encryption (in x86 version they are XORed after).
+
+inline Vector128 Encrypt128(Vector128 data, Vector128 key) {
+  return vaesmcq_u8(vaeseq_u8(data, key));
+}
+
+inline Vector128 Decrypt128(Vector128 data, Vector128 key) {
+  return vaesimcq_u8(vaesdq_u8(data, key));
+}
+
+// We use decryption for a, b and encryption for c, d. That helps us to avoid
+// collisions for trivial byte rotations. Mix4x16Vectors later uses
+// encrypted/decrypted pairs differently to ensure that the order of blocks is
+// important for the hash value.
+// We also avoid using Add128/Sub128 instructions because state is being mixed
+// before encryption/decryption. On ARM, there is no fusion of load and add/sub
+// instructions so it is more expensive to use them.
+
+inline Vector128 MixA(Vector128 a, Vector128 state) {
+  return Decrypt128(a, state);
+}
+
+inline Vector128 MixB(Vector128 b, Vector128 state) {
+  return Decrypt128(b, state);
+}
+
+inline Vector128 MixC(Vector128 c, Vector128 state) {
+  return Encrypt128(c, state);
+}
+
+inline Vector128 MixD(Vector128 d, Vector128 state) {
+  return Encrypt128(d, state);
+}
+
+inline uint64_t ExtractLow64(Vector128 v) {
+  return vgetq_lane_u64(vreinterpretq_u64_u8(v), 0);
+}
+
+inline uint64_t ExtractHigh64(Vector128 v) {
+  return vgetq_lane_u64(vreinterpretq_u64_u8(v), 1);
+}
+
+uint64_t Mix4x16Vectors(Vector128 a, Vector128 b, Vector128 c, Vector128 d) {
+  Vector128 res128 = Add128(Encrypt128(a, c), Decrypt128(b, d));
+  uint64_t x64 = ExtractLow64(res128);
+  uint64_t y64 = ExtractHigh64(res128);
+  return x64 ^ y64;
+}
+
+#endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
+
 uint64_t LowLevelHash33To64(uint64_t seed, const uint8_t* ptr, size_t len) {
   assert(len > 32);
   assert(len <= 64);
-  __m128i state =
-      _mm_set_epi64x(static_cast<int64_t>(seed), static_cast<int64_t>(len));
-  auto a = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr));
-  auto b = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr + 16));
+  Vector128 state = Set128(seed, len);
+  Vector128 a = Load128(ptr);
+  Vector128 b = Load128(ptr + 16);
   auto* last32_ptr = ptr + len - 32;
-  auto c = _mm_loadu_si128(reinterpret_cast<const __m128i*>(last32_ptr));
-  auto d = _mm_loadu_si128(reinterpret_cast<const __m128i*>(last32_ptr + 16));
-
-  // Bits of the second argument to _mm_aesdec_si128/_mm_aesenc_si128 are
-  // XORed with the state argument after encryption.
-  // We use each value as the first argument to shuffle all the bits around.
-  // We do not add any salt to the state or loaded data, instead we vary
-  // instructions used to mix bits _mm_aesdec_si128/_mm_aesenc_si128 and
-  // _mm_add_epi64/_mm_sub_epi64.
-  // _mm_add_epi64/_mm_sub_epi64 are combined to one instruction with data
-  // loading like `vpaddq  xmm1, xmm0, xmmword ptr [rdi]`.
-  auto na = _mm_aesdec_si128(_mm_add_epi64(state, a), state);
-  auto nb = _mm_aesdec_si128(_mm_sub_epi64(state, b), state);
-  auto nc = _mm_aesenc_si128(_mm_add_epi64(state, c), state);
-  auto nd = _mm_aesenc_si128(_mm_sub_epi64(state, d), state);
+  Vector128 c = Load128(last32_ptr);
+  Vector128 d = Load128(last32_ptr + 16);
+
+  Vector128 na = MixA(a, state);
+  Vector128 nb = MixB(b, state);
+  Vector128 nc = MixC(c, state);
+  Vector128 nd = MixD(d, state);
 
   // We perform another round of encryption to mix bits between two halves of
   // the input.
@@ -97,11 +224,10 @@ LowLevelHashLenGt64(uint64_t seed, const void* data, size_t len) {
   // If we have more than 64 bytes, we're going to handle chunks of 64
   // bytes at a time. We're going to build up four separate hash states
   // which we will then hash together. This avoids short dependency chains.
-  __m128i state0 =
-      _mm_set_epi64x(static_cast<int64_t>(seed), static_cast<int64_t>(len));
-  __m128i state1 = state0;
-  __m128i state2 = state1;
-  __m128i state3 = state2;
+  Vector128 state0 = Set128(seed, len);
+  Vector128 state1 = state0;
+  Vector128 state2 = state1;
+  Vector128 state3 = state2;
 
   // Mixing two 128-bit vectors at a time with corresponding states.
   // All variables are mixed slightly differently to avoid hash collision
@@ -111,25 +237,17 @@ LowLevelHashLenGt64(uint64_t seed, const void* data, size_t len) {
   // See comments in LowLevelHash33To64 for more considerations.
   auto mix_ab = [&state0,
                  &state1](const uint8_t* p) ABSL_ATTRIBUTE_ALWAYS_INLINE {
-    // i128 a = *p;
-    // i128 b = *(p + 16);
-    // state0 = decrypt(state0 + a, state0);
-    // state1 = decrypt(state1 - b, state1);
-    auto a = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
-    auto b = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + 16));
-    state0 = _mm_aesdec_si128(_mm_add_epi64(state0, a), state0);
-    state1 = _mm_aesdec_si128(_mm_sub_epi64(state1, b), state1);
+    Vector128 a = Load128(p);
+    Vector128 b = Load128(p + 16);
+    state0 = MixA(a, state0);
+    state1 = MixB(b, state1);
   };
   auto mix_cd = [&state2,
                  &state3](const uint8_t* p) ABSL_ATTRIBUTE_ALWAYS_INLINE {
-    // i128 c = *p;
-    // i128 d = *(p + 16);
-    // state2 = encrypt(state2 + c, state2);
-    // state3 = encrypt(state3 - d, state3);
-    auto c = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
-    auto d = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + 16));
-    state2 = _mm_aesenc_si128(_mm_add_epi64(state2, c), state2);
-    state3 = _mm_aesenc_si128(_mm_sub_epi64(state3, d), state3);
+    Vector128 c = Load128(p);
+    Vector128 d = Load128(p + 16);
+    state2 = MixC(c, state2);
+    state3 = MixD(d, state3);
   };
 
   do {