react-native-windows 0.84.0 → 0.84.1

package/external/folly/folly/concurrency/UnboundedQueue.h

@@ -0,0 +1,891 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <memory>
+
+#include <glog/logging.h>
+
+#include <folly/ConstexprMath.h>
+#include <folly/Optional.h>
+#include <folly/Traits.h>
+#include <folly/concurrency/CacheLocality.h>
+#include <folly/lang/Align.h>
+#include <folly/synchronization/Hazptr.h>
+#include <folly/synchronization/SaturatingSemaphore.h>
+#include <folly/synchronization/WaitOptions.h>
+#include <folly/synchronization/detail/Spin.h>
+
+namespace folly {
+
+/// UnboundedQueue supports a variety of options for unbounded
+/// dynamically expanding an shrinking queues, including variations of:
+/// - Single vs. multiple producers
+/// - Single vs. multiple consumers
+/// - Blocking vs. spin-waiting
+/// - Non-waiting, timed, and waiting consumer operations.
+/// Producer operations never wait or fail (unless out-of-memory).
+///
+/// Template parameters:
+/// - T: element type
+/// - SingleProducer: true if there can be only one producer at a
+///   time.
+/// - SingleConsumer: true if there can be only one consumer at a
+///   time.
+/// - MayBlock: true if consumers may block, false if they only
+///   spin. A performance tuning parameter.
+/// - LgSegmentSize (default 8): Log base 2 of number of elements per
+///   segment. A performance tuning parameter. See below.
+/// - LgAlign (default 7): Log base 2 of alignment directive; can be
+///   used to balance scalability (avoidance of false sharing) with
+///   memory efficiency.
+///
+/// When to use UnboundedQueue:
+/// - If a small bound may lead to deadlock or performance degradation
+///   under bursty patterns.
+/// - If there is no risk of the queue growing too much.
+///
+/// When not to use UnboundedQueue:
+/// - If there is risk of the queue growing too much and a large bound
+///   is acceptable, then use DynamicBoundedQueue.
+/// - If the queue must not allocate on enqueue or it must have a
+///   small bound, then use fixed-size MPMCQueue or (if non-blocking
+///   SPSC) ProducerConsumerQueue.
+///
+/// Template Aliases:
+///   USPSCQueue<T, MayBlock, LgSegmentSize, LgAlign>
+///   UMPSCQueue<T, MayBlock, LgSegmentSize, LgAlign>
+///   USPMCQueue<T, MayBlock, LgSegmentSize, LgAlign>
+///   UMPMCQueue<T, MayBlock, LgSegmentSize, LgAlign>
+///
+/// Functions:
+///   Producer operations never wait or fail (unless OOM)
+///     void enqueue(const T&);
+///     void enqueue(T&&);
+///         Adds an element to the end of the queue.
+///
+///   Consumer operations:
+///     void dequeue(T&);
+///     T dequeue();
+///         Extracts an element from the front of the queue. Waits
+///         until an element is available if needed.
+///     bool try_dequeue(T&);
+///     folly::Optional<T> try_dequeue();
+///         Tries to extract an element from the front of the queue
+///         if available.
+///     bool try_dequeue_until(T&, time_point& deadline);
+///     folly::Optional<T> try_dequeue_until(time_point& deadline);
+///         Tries to extract an element from the front of the queue
+///         if available until the specified deadline.
+///     bool try_dequeue_for(T&, duration&);
+///     folly::Optional<T> try_dequeue_for(duration&);
+///         Tries to extract an element from the front of the queue if
+///         available until the expiration of the specified duration.
+///     const T* try_peek();
+///         Returns pointer to the element at the front of the queue
+///         if available, or nullptr if the queue is empty. Only for
+///         SPSC and MPSC.
+///
+///   Secondary functions:
+///     size_t size();
+///         Returns an estimate of the size of the queue.
+///     bool empty();
+///         Returns true only if the queue was empty during the call.
+///     Note: size() and empty() are guaranteed to be accurate only if
+///     the queue is not changed concurrently.
+///
+/// Usage examples:
+/// @code
+///   /* UMPSC, doesn't block, 1024 int elements per segment */
+///   UMPSCQueue<int, false, 10> q;
+///   q.enqueue(1);
+///   q.enqueue(2);
+///   q.enqueue(3);
+///   ASSERT_FALSE(q.empty());
+///   ASSERT_EQ(q.size(), 3);
+///   int v;
+///   q.dequeue(v);
+///   ASSERT_EQ(v, 1);
+///   ASSERT_TRUE(try_dequeue(v));
+///   ASSERT_EQ(v, 2);
+///   ASSERT_TRUE(try_dequeue_until(v, now() + seconds(1)));
+///   ASSERT_EQ(v, 3);
+///   ASSERT_TRUE(q.empty());
+///   ASSERT_EQ(q.size(), 0);
+///   ASSERT_FALSE(try_dequeue(v));
+///   ASSERT_FALSE(try_dequeue_for(v, microseconds(100)));
+/// @endcode
+///
+/// Design:
+/// - The queue is composed of one or more segments. Each segment has
+///   a fixed size of 2^LgSegmentSize entries. Each segment is used
+///   exactly once.
+/// - Each entry is composed of a futex and a single element.
+/// - Each segment's array of entries is strided to avoid false sharing.
+///   I.e., to reduce any cacheline contention that might be induced by
+///   concurrent mutations to the queue that might happen to affect
+///   otherwise-adjacent locations that might happen to share cacheline.
+/// - The queue contains two 64-bit ticket variables. The producer
+///   ticket counts the number of producer tickets issued so far, and
+///   the same for the consumer ticket. Each ticket number corresponds
+///   to a specific entry in a specific segment.
+/// - The queue maintains two pointers, head and tail. Head points to
+///   the segment that corresponds to the current consumer
+///   ticket. Similarly, tail pointer points to the segment that
+///   corresponds to the producer ticket.
+/// - Segments are organized as a singly linked list.
+/// - The producer with the first ticket in the current producer
+///   segment has primary responsibility for allocating and linking
+///   the next segment. Other producers and connsumers may help do so
+///   when needed if that thread is delayed.
+/// - The producer with the last ticket in the current producer
+///   segment is primarily responsible for advancing the tail pointer
+///   to the next segment. Other producers and consumers may help do
+///   so when needed if that thread is delayed.
+/// - Similarly, the consumer with the last ticket in the current
+///   consumer segment is primarily responsible for advancing the head
+///   pointer to the next segment. Other consumers may help do so when
+///   needed if that thread is delayed.
+/// - The tail pointer must not lag behind the head pointer.
+///   Otherwise, the algorithm cannot be certain about the removal of
+///   segment and would have to incur higher costs to ensure safe
+///   reclamation.  Consumers must ensure that head never overtakes
+///   tail.
+///
+/// Memory Usage:
+/// - An empty queue contains one segment. A nonempty queue contains
+///   one or two more segment than fits its contents.
+/// - Removed segments are not reclaimed until there are no threads,
+///   producers or consumers, with references to them or their
+///   predecessors. That is, a lagging thread may delay the reclamation
+///   of a chain of removed segments.
+/// - The template parameter LgAlign can be used to reduce memory usage
+///   at the cost of increased chance of false sharing.
+///
+/// Performance considerations:
+/// - All operations take constant time, excluding the costs of
+///   allocation, reclamation, interference from other threads, and
+///   waiting for actions by other threads.
+/// - In general, using the single producer and or single consumer
+///   variants yield better performance than the MP and MC
+///   alternatives.
+/// - SPSC without blocking is the fastest configuration. It doesn't
+///   include any read-modify-write atomic operations, full fences, or
+///   system calls in the critical path.
+/// - MP adds a fetch_add to the critical path of each producer operation.
+/// - MC adds a fetch_add or compare_exchange to the critical path of
+///   each consumer operation.
+/// - The possibility of consumers blocking, even if they never do,
+///   adds a compare_exchange to the critical path of each producer
+///   operation.
+/// - MPMC, SPMC, MPSC require the use of a deferred reclamation
+///   mechanism to guarantee that segments removed from the linked
+///   list, i.e., unreachable from the head pointer, are reclaimed
+///   only after they are no longer needed by any lagging producers or
+///   consumers.
+/// - The overheads of segment allocation and reclamation are intended
+///   to be mostly out of the critical path of the queue's throughput.
+/// - If the template parameter LgSegmentSize is changed, it should be
+///   set adequately high to keep the amortized cost of allocation and
+///   reclamation low.
+/// - It is recommended to measure performance with different variants
+///   when applicable, e.g., UMPMC vs UMPSC. Depending on the use
+///   case, sometimes the variant with the higher sequential overhead
+///   may yield better results due to, for example, more favorable
+///   producer-consumer balance or favorable timing for avoiding
+///   costly blocking.
+///
+/// Guarantees:
+/// - The queues are linearizable:
+///   - For two enqueue operations q(A) and q(B), if q(A) < q(B) in
+///     the happens-before relation, then A precedes B in the queue.
+///   - For two dequeue operations d(A) and d(B), if d(A) < d(B) in
+///     the happens-before relation, then A preceded B in the queue.
+
+template <
+    typename T,
+    bool SingleProducer,
+    bool SingleConsumer,
+    bool MayBlock,
+    size_t LgSegmentSize = 8,
+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),
+    template <typename> class Atom = std::atomic>
+class UnboundedQueue {
+  using Ticket = uint64_t;
+  class Entry;
+  class Segment;
+
+  static constexpr bool SPSC = SingleProducer && SingleConsumer;
+  static constexpr size_t Stride = SPSC || (LgSegmentSize <= 1) ? 1 : 27;
+  static constexpr size_t SegmentSize = 1u << LgSegmentSize;
+  static constexpr size_t Align = 1u << LgAlign;
+
+  static_assert(
+      std::is_nothrow_destructible<T>::value, "T must be nothrow_destructible");
+  static_assert((Stride & 1) == 1, "Stride must be odd");
+  static_assert(LgSegmentSize < 32, "LgSegmentSize must be < 32");
+  static_assert(LgAlign < 16, "LgAlign must be < 16");
+
+  using Sem = folly::SaturatingSemaphore<MayBlock, Atom>;
+
+  struct Consumer {
+    Atom<Segment*> head;
+    Atom<Ticket> ticket;
+    hazptr_obj_cohort<Atom> cohort;
+    explicit Consumer(Segment* s) : head(s), ticket(0) {
+      s->set_cohort_no_tag(&cohort); // defined in hazptr_obj
+    }
+  };
+  struct Producer {
+    Atom<Segment*> tail;
+    Atom<Ticket> ticket;
+    explicit Producer(Segment* s) : tail(s), ticket(0) {}
+  };
+
+  alignas(Align) Consumer c_;
+  alignas(Align) Producer p_;
+
+ public:
+  using value_type = T;
+  using size_type = size_t;
+
+  /** constructor */
+  UnboundedQueue()
+      : c_(new Segment(0)), p_(c_.head.load(std::memory_order_relaxed)) {}
+
+  /** destructor */
+  ~UnboundedQueue() {
+    cleanUpRemainingItems();
+    reclaimRemainingSegments();
+  }
+
+  /** enqueue */
+  FOLLY_ALWAYS_INLINE void enqueue(const T& arg) { enqueueImpl(arg); }
+
+  FOLLY_ALWAYS_INLINE void enqueue(T&& arg) { enqueueImpl(std::move(arg)); }
+
+  /** dequeue */
+  FOLLY_ALWAYS_INLINE void dequeue(T& item) noexcept { item = dequeueImpl(); }
+
+  FOLLY_ALWAYS_INLINE T dequeue() noexcept { return dequeueImpl(); }
+
+  /** try_dequeue */
+  FOLLY_ALWAYS_INLINE bool try_dequeue(T& item) noexcept {
+    auto o = try_dequeue();
+    if (FOLLY_LIKELY(o.has_value())) {
+      item = std::move(*o);
+      return true;
+    }
+    return false;
+  }
+
+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue() noexcept {
+    return tryDequeueUntil(std::chrono::steady_clock::time_point::min());
+  }
+
+  /** try_dequeue_until */
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE bool try_dequeue_until(
+      T& item,
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    folly::Optional<T> o = try_dequeue_until(deadline);
+
+    if (FOLLY_LIKELY(o.has_value())) {
+      item = std::move(*o);
+      return true;
+    }
+
+    return false;
+  }
+
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue_until(
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    return tryDequeueUntil(deadline);
+  }
+
+  /** try_dequeue_for */
+  template <typename Rep, typename Period>
+  FOLLY_ALWAYS_INLINE bool try_dequeue_for(
+      T& item, const std::chrono::duration<Rep, Period>& duration) noexcept {
+    folly::Optional<T> o = try_dequeue_for(duration);
+
+    if (FOLLY_LIKELY(o.has_value())) {
+      item = std::move(*o);
+      return true;
+    }
+
+    return false;
+  }
+
+  template <typename Rep, typename Period>
+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue_for(
+      const std::chrono::duration<Rep, Period>& duration) noexcept {
+    folly::Optional<T> o = try_dequeue();
+    if (FOLLY_LIKELY(o.has_value())) {
+      return o;
+    }
+    return tryDequeueUntil(std::chrono::steady_clock::now() + duration);
+  }
+
+  /** try_peek */
+  FOLLY_ALWAYS_INLINE const T* try_peek() noexcept {
+    static_assert(SingleConsumer, "not single-consumer");
+    return tryPeekUntil(std::chrono::steady_clock::time_point::min());
+  }
+
+  /** size */
+  size_t size() const noexcept {
+    auto p = producerTicket();
+    auto c = consumerTicket();
+    return p > c ? p - c : 0;
+  }
+
+  /** empty */
+  bool empty() const noexcept {
+    auto c = consumerTicket();
+    auto p = producerTicket();
+    return p <= c;
+  }
+
+ private:
+  /** enqueueImpl */
+  template <typename Arg>
+  FOLLY_ALWAYS_INLINE void enqueueImpl(Arg&& arg) {
+    if (SPSC) {
+      Segment* s = tail();
+      enqueueCommon(s, std::forward<Arg>(arg));
+    } else {
+      // Using hazptr_holder instead of hazptr_local because it is
+      // possible that the T ctor happens to use hazard pointers.
+      hazptr_holder<Atom> hptr = make_hazard_pointer<Atom>();
+      Segment* s = hptr.protect(p_.tail);
+      enqueueCommon(s, std::forward<Arg>(arg));
+    }
+  }
+
+  /** enqueueCommon */
+  template <typename Arg>
+  FOLLY_ALWAYS_INLINE void enqueueCommon(Segment* s, Arg&& arg) {
+    Ticket t = fetchIncrementProducerTicket();
+    if (!SingleProducer) {
+      s = findSegment(s, t);
+    }
+    DCHECK_GE(t, s->minTicket());
+    DCHECK_LT(t, s->minTicket() + SegmentSize);
+    size_t idx = index(t);
+    Entry& e = s->entry(idx);
+    e.putItem(std::forward<Arg>(arg));
+    if (responsibleForAlloc(t)) {
+      allocNextSegment(s);
+    }
+    if (responsibleForAdvance(t)) {
+      advanceTail(s);
+    }
+  }
+
+  /** dequeueImpl */
+  FOLLY_ALWAYS_INLINE T dequeueImpl() noexcept {
+    if (SPSC) {
+      Segment* s = head();
+      return dequeueCommon(s);
+    } else {
+      // Using hazptr_holder instead of hazptr_local because it is
+      // possible to call the T dtor and it may happen to use hazard
+      // pointers.
+      hazptr_holder<Atom> hptr = make_hazard_pointer<Atom>();
+      Segment* s = hptr.protect(c_.head);
+      return dequeueCommon(s);
+    }
+  }
+
+  /** dequeueCommon */
+  FOLLY_ALWAYS_INLINE T dequeueCommon(Segment* s) noexcept {
+    Ticket t = fetchIncrementConsumerTicket();
+    if (!SingleConsumer) {
+      s = findSegment(s, t);
+    }
+    size_t idx = index(t);
+    Entry& e = s->entry(idx);
+    auto res = e.takeItem();
+    if (responsibleForAdvance(t)) {
+      advanceHead(s);
+    }
+    return res;
+  }
+
+  /** tryDequeueUntil */
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE folly::Optional<T> tryDequeueUntil(
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    if (SingleConsumer) {
+      Segment* s = head();
+      return tryDequeueUntilSC(s, deadline);
+    } else {
+      // Using hazptr_holder instead of hazptr_local because it is
+      //  possible to call ~T() and it may happen to use hazard pointers.
+      hazptr_holder<Atom> hptr = make_hazard_pointer<Atom>();
+      Segment* s = hptr.protect(c_.head);
+      return tryDequeueUntilMC(s, deadline);
+    }
+  }
+
+  /** tryDequeueUntilSC */
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE folly::Optional<T> tryDequeueUntilSC(
+      Segment* s,
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    Ticket t = consumerTicket();
+    DCHECK_GE(t, s->minTicket());
+    DCHECK_LT(t, (s->minTicket() + SegmentSize));
+    size_t idx = index(t);
+    Entry& e = s->entry(idx);
+    if (FOLLY_UNLIKELY(!tryDequeueWaitElem(e, t, deadline))) {
+      return folly::Optional<T>();
+    }
+    setConsumerTicket(t + 1);
+    folly::Optional<T> ret = e.takeItem();
+    if (responsibleForAdvance(t)) {
+      advanceHead(s);
+    }
+    return ret;
+  }
+
+  /** tryDequeueUntilMC */
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE folly::Optional<T> tryDequeueUntilMC(
+      Segment* s,
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    while (true) {
+      Ticket t = consumerTicket();
+      if (FOLLY_UNLIKELY(t >= (s->minTicket() + SegmentSize))) {
+        s = getAllocNextSegment(s, t);
+        DCHECK(s);
+        continue;
+      }
+      size_t idx = index(t);
+      Entry& e = s->entry(idx);
+      if (FOLLY_UNLIKELY(!tryDequeueWaitElem(e, t, deadline))) {
+        return folly::Optional<T>();
+      }
+      if (!c_.ticket.compare_exchange_weak(
+              t, t + 1, std::memory_order_acq_rel, std::memory_order_acquire)) {
+        continue;
+      }
+      folly::Optional<T> ret = e.takeItem();
+      if (responsibleForAdvance(t)) {
+        advanceHead(s);
+      }
+      return ret;
+    }
+  }
+
+  /** tryDequeueWaitElem */
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE bool tryDequeueWaitElem(
+      Entry& e,
+      Ticket t,
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    if (FOLLY_LIKELY(e.tryWaitUntil(deadline))) {
+      return true;
+    }
+    return t < producerTicket();
+  }
+
+  /** tryPeekUntil */
+  template <typename Clock, typename Duration>
+  FOLLY_ALWAYS_INLINE const T* tryPeekUntil(
+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+    // This function is supported only for USPSC and UMPSC queues.
+    DCHECK(SingleConsumer);
+    Segment* s = head();
+    Ticket t = consumerTicket();
+    DCHECK_GE(t, s->minTicket());
+    DCHECK_LT(t, (s->minTicket() + SegmentSize));
+    size_t idx = index(t);
+    Entry& e = s->entry(idx);
+    if (FOLLY_UNLIKELY(!tryDequeueWaitElem(e, t, deadline))) {
+      return nullptr;
+    }
+    return e.peekItem();
+  }
+
+  /** findSegment */
+  FOLLY_ALWAYS_INLINE
+  Segment* findSegment(Segment* s, const Ticket t) noexcept {
+    while (FOLLY_UNLIKELY(t >= (s->minTicket() + SegmentSize))) {
+      s = getAllocNextSegment(s, t);
+      DCHECK(s);
+    }
+    return s;
+  }
+
+  /** getAllocNextSegment */
+  Segment* getAllocNextSegment(Segment* s, Ticket t) noexcept {
+    Segment* next = s->nextSegment();
+    if (!next) {
+      DCHECK_GE(t, s->minTicket() + SegmentSize);
+      auto diff = t - (s->minTicket() + SegmentSize);
+      if (diff > 0) {
+        auto dur = std::chrono::microseconds(diff);
+        auto deadline = std::chrono::steady_clock::now() + dur;
+        WaitOptions opt;
+        opt.spin_max(dur);
+        detail::spin_pause_until(
+            deadline, opt, [s] { return s->nextSegment(); });
+        next = s->nextSegment();
+        if (next) {
+          return next;
+        }
+      }
+      next = allocNextSegment(s);
+    }
+    DCHECK(next);
+    return next;
+  }
+
+  /** allocNextSegment */
+  Segment* allocNextSegment(Segment* s) {
+    auto t = s->minTicket() + SegmentSize;
+    Segment* next = new Segment(t);
+    next->set_cohort_no_tag(&c_.cohort); // defined in hazptr_obj
+    next->acquire_ref_safe(); // defined in hazptr_obj_base_linked
+    if (!s->casNextSegment(next)) {
+      delete next;
+      next = s->nextSegment();
+    }
+    DCHECK(next);
+    return next;
+  }
+
+  /** advanceTail */
+  void advanceTail(Segment* s) noexcept {
+    if (SPSC) {
+      Segment* next = s->nextSegment();
+      DCHECK(next);
+      setTail(next);
+    } else {
+      Ticket t = s->minTicket() + SegmentSize;
+      advanceTailToTicket(t);
+    }
+  }
+
+  /** advanceTailToTicket */
+  void advanceTailToTicket(Ticket t) noexcept {
+    Segment* s = tail();
+    while (s->minTicket() < t) {
+      Segment* next = s->nextSegment();
+      if (!next) {
+        next = allocNextSegment(s);
+      }
+      DCHECK(next);
+      casTail(s, next);
+      s = tail();
+    }
+  }
+
+  /** advanceHead */
+  void advanceHead(Segment* s) noexcept {
+    if (SPSC) {
+      while (tail() == s) {
+        /* Wait for producer to advance tail. */
+        asm_volatile_pause();
+      }
+      Segment* next = s->nextSegment();
+      DCHECK(next);
+      setHead(next);
+      reclaimSegment(s);
+    } else {
+      Ticket t = s->minTicket() + SegmentSize;
+      advanceHeadToTicket(t);
+    }
+  }
+
+  /** advanceHeadToTicket */
+  void advanceHeadToTicket(Ticket t) noexcept {
+    /* Tail must not lag behind head. Otherwise, the algorithm cannot
+       be certain about removal of segments. */
+    advanceTailToTicket(t);
+    Segment* s = head();
+    if (SingleConsumer) {
+      DCHECK_EQ(s->minTicket() + SegmentSize, t);
+      Segment* next = s->nextSegment();
+      DCHECK(next);
+      setHead(next);
+      reclaimSegment(s);
+    } else {
+      while (s->minTicket() < t) {
+        Segment* next = s->nextSegment();
+        DCHECK(next);
+        if (casHead(s, next)) {
+          reclaimSegment(s);
+          s = next;
+        }
+      }
+    }
+  }
+
+  /** reclaimSegment */
+  void reclaimSegment(Segment* s) noexcept {
+    if (SPSC) {
+      delete s;
+    } else {
+      s->retire(); // defined in hazptr_obj_base_linked
+    }
+  }
+
+  /** cleanUpRemainingItems */
+  void cleanUpRemainingItems() {
+    auto end = producerTicket();
+    auto s = head();
+    for (auto t = consumerTicket(); t < end; ++t) {
+      if (t >= s->minTicket() + SegmentSize) {
+        s = s->nextSegment();
+      }
+      DCHECK_LT(t, (s->minTicket() + SegmentSize));
+      auto idx = index(t);
+      auto& e = s->entry(idx);
+      e.destroyItem();
+    }
+  }
+
+  /** reclaimRemainingSegments */
+  void reclaimRemainingSegments() {
+    auto h = head();
+    auto s = h->nextSegment();
+    h->setNextSegment(nullptr);
+    reclaimSegment(h);
+    while (s) {
+      auto next = s->nextSegment();
+      delete s;
+      s = next;
+    }
+  }
+
+  FOLLY_ALWAYS_INLINE size_t index(Ticket t) const noexcept {
+    return (t * Stride) & (SegmentSize - 1);
+  }
+
+  FOLLY_ALWAYS_INLINE bool responsibleForAlloc(Ticket t) const noexcept {
+    return (t & (SegmentSize - 1)) == 0;
+  }
+
+  FOLLY_ALWAYS_INLINE bool responsibleForAdvance(Ticket t) const noexcept {
+    return (t & (SegmentSize - 1)) == (SegmentSize - 1);
+  }
+
+  FOLLY_ALWAYS_INLINE Segment* head() const noexcept {
+    return c_.head.load(std::memory_order_acquire);
+  }
+
+  FOLLY_ALWAYS_INLINE Segment* tail() const noexcept {
+    return p_.tail.load(std::memory_order_acquire);
+  }
+
+  FOLLY_ALWAYS_INLINE Ticket producerTicket() const noexcept {
+    return p_.ticket.load(std::memory_order_acquire);
+  }
+
+  FOLLY_ALWAYS_INLINE Ticket consumerTicket() const noexcept {
+    return c_.ticket.load(std::memory_order_acquire);
+  }
+
+  void setHead(Segment* s) noexcept {
+    DCHECK(SingleConsumer);
+    c_.head.store(s, std::memory_order_relaxed);
+  }
+
+  void setTail(Segment* s) noexcept {
+    DCHECK(SPSC);
+    p_.tail.store(s, std::memory_order_release);
+  }
+
+  bool casHead(Segment*& s, Segment* next) noexcept {
+    DCHECK(!SingleConsumer);
+    return c_.head.compare_exchange_strong(
+        s, next, std::memory_order_release, std::memory_order_acquire);
+  }
+
+  void casTail(Segment*& s, Segment* next) noexcept {
+    DCHECK(!SPSC);
+    p_.tail.compare_exchange_strong(
+        s, next, std::memory_order_release, std::memory_order_relaxed);
+  }
+
+  FOLLY_ALWAYS_INLINE void setProducerTicket(Ticket t) noexcept {
+    p_.ticket.store(t, std::memory_order_release);
+  }
+
+  FOLLY_ALWAYS_INLINE void setConsumerTicket(Ticket t) noexcept {
+    c_.ticket.store(t, std::memory_order_release);
+  }
+
+  FOLLY_ALWAYS_INLINE Ticket fetchIncrementConsumerTicket() noexcept {
+    if (SingleConsumer) {
+      Ticket oldval = consumerTicket();
+      setConsumerTicket(oldval + 1);
+      return oldval;
+    } else { // MC
+      return c_.ticket.fetch_add(1, std::memory_order_acq_rel);
+    }
+  }
+
+  FOLLY_ALWAYS_INLINE Ticket fetchIncrementProducerTicket() noexcept {
+    if (SingleProducer) {
+      Ticket oldval = producerTicket();
+      setProducerTicket(oldval + 1);
+      return oldval;
+    } else { // MP
+      return p_.ticket.fetch_add(1, std::memory_order_acq_rel);
+    }
+  }
+
+  /**
+   *  Entry
+   */
+  class Entry {
+    Sem flag_;
+    aligned_storage_for_t<T> item_;
+
+   public:
+    template <typename Arg>
+    FOLLY_ALWAYS_INLINE void putItem(Arg&& arg) {
+      new (&item_) T(std::forward<Arg>(arg));
+      flag_.post();
+    }
+
+    FOLLY_ALWAYS_INLINE T takeItem() noexcept {
+      flag_.wait();
+      return getItem();
+    }
+
+    FOLLY_ALWAYS_INLINE const T* peekItem() noexcept {
+      flag_.wait();
+      return itemPtr();
+    }
+
+    template <typename Clock, typename Duration>
+    FOLLY_EXPORT FOLLY_ALWAYS_INLINE bool tryWaitUntil(
+        const std::chrono::time_point<Clock, Duration>& deadline) noexcept {
+      // wait-options from benchmarks on contended queues:
+      static constexpr auto const opt =
+          Sem::wait_options().spin_max(std::chrono::microseconds(10));
+      return flag_.try_wait_until(deadline, opt);
+    }
+
+    FOLLY_ALWAYS_INLINE void destroyItem() noexcept { itemPtr()->~T(); }
+
+   private:
+    FOLLY_ALWAYS_INLINE T getItem() noexcept {
+      T ret = std::move(*(itemPtr()));
+      destroyItem();
+      return ret;
+    }
+
+    FOLLY_ALWAYS_INLINE T* itemPtr() noexcept {
+      return static_cast<T*>(static_cast<void*>(&item_));
+    }
+  }; // Entry
+
+  /**
+   *  Segment
+   */
+  class Segment : public hazptr_obj_base_linked<Segment, Atom> {
+    Atom<Segment*> next_{nullptr};
+    const Ticket min_;
+    alignas(Align) Entry b_[SegmentSize];
+
+   public:
+    explicit Segment(const Ticket t) noexcept : min_(t) {}
+
+    Segment* nextSegment() const noexcept {
+      return next_.load(std::memory_order_acquire);
+    }
+
+    void setNextSegment(Segment* next) {
+      next_.store(next, std::memory_order_relaxed);
+    }
+
+    bool casNextSegment(Segment* next) noexcept {
+      Segment* expected = nullptr;
+      return next_.compare_exchange_strong(
+          expected, next, std::memory_order_release, std::memory_order_relaxed);
+    }
+
+    FOLLY_ALWAYS_INLINE Ticket minTicket() const noexcept {
+      DCHECK_EQ((min_ & (SegmentSize - 1)), Ticket(0));
+      return min_;
+    }
+
+    FOLLY_ALWAYS_INLINE Entry& entry(size_t index) noexcept {
+      return b_[index];
+    }
+
+    template <typename S>
+    void push_links(bool m, S& s) {
+      if (m == false) { // next_ is immutable
+        auto p = nextSegment();
+        if (p) {
+          s.push(p);
+        }
+      }
+    }
+  }; // Segment
+
+}; // UnboundedQueue
+
+/* Aliases */
+
+template <
+    typename T,
+    bool MayBlock,
+    size_t LgSegmentSize = 8,
+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),
+    template <typename> class Atom = std::atomic>
+using USPSCQueue =
+    UnboundedQueue<T, true, true, MayBlock, LgSegmentSize, LgAlign, Atom>;
+
+template <
+    typename T,
+    bool MayBlock,
+    size_t LgSegmentSize = 8,
+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),
+    template <typename> class Atom = std::atomic>
+using UMPSCQueue =
+    UnboundedQueue<T, false, true, MayBlock, LgSegmentSize, LgAlign, Atom>;
+
+template <
+    typename T,
+    bool MayBlock,
+    size_t LgSegmentSize = 8,
+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),
+    template <typename> class Atom = std::atomic>
+using USPMCQueue =
+    UnboundedQueue<T, true, false, MayBlock, LgSegmentSize, LgAlign, Atom>;
+
+template <
+    typename T,
+    bool MayBlock,
+    size_t LgSegmentSize = 8,
+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),
+    template <typename> class Atom = std::atomic>
+using UMPMCQueue =
+    UnboundedQueue<T, false, false, MayBlock, LgSegmentSize, LgAlign, Atom>;
+
+} // namespace folly