12 #include <type_traits> 15 #if __cplusplus > 199711L || _MSC_VER >= 1700 // C++11 or VS2012 32 #ifndef MOODYCAMEL_CACHE_LINE_SIZE 33 #define MOODYCAMEL_CACHE_LINE_SIZE 64 36 #ifndef MOODYCAMEL_EXCEPTIONS_ENABLED 37 #if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || \ 38 (!defined(_MSC_VER) && !defined(__GNUC__)) 39 #define MOODYCAMEL_EXCEPTIONS_ENABLED 45 #pragma warning(disable : 4324) // structure was padded due to __declspec(align()) 46 #pragma warning(disable : 4820) // padding was added 47 #pragma warning(disable : 4127) // conditional expression is constant 52 template <
typename T,
size_t MAX_BLOCK_SIZE = 512>
86 assert(MAX_BLOCK_SIZE ==
ceilToPow2(MAX_BLOCK_SIZE) &&
"MAX_BLOCK_SIZE must be a power of 2");
87 assert(MAX_BLOCK_SIZE >= 2 &&
"MAX_BLOCK_SIZE must be at least 2");
89 Block* first_block =
nullptr;
100 size_t initial_block_count = (max_size + MAX_BLOCK_SIZE * 2 - 3) / (MAX_BLOCK_SIZE - 1);
102 Block* last_block =
nullptr;
103 for (
size_t i = 0; i != initial_block_count; ++i)
106 if (block ==
nullptr)
108 #ifdef MOODYCAMEL_EXCEPTIONS_ENABLED 109 throw std::bad_alloc();
114 if (first_block ==
nullptr)
120 last_block->next = block;
123 block->next = first_block;
129 if (first_block ==
nullptr)
131 #ifdef MOODYCAMEL_EXCEPTIONS_ENABLED 132 throw std::bad_alloc();
137 first_block->next = first_block;
155 Block* block = front_block;
158 Block* next_block = block->next;
159 size_t block_front = block->front;
160 size_t block_tail = block->tail;
162 for (
size_t i = block_front; i != block_tail; i = (i + 1) & block->sizeMask)
164 auto element =
reinterpret_cast<T*
>(block->data + i *
sizeof(T));
169 auto raw_block = block->rawThis;
171 std::free(raw_block);
173 }
while (block != front_block);
181 return innerEnqueue<CannotAlloc>(element);
189 return innerEnqueue<CannotAlloc>(std::forward<T>(element));
197 return innerEnqueue<CanAlloc>(element);
205 return innerEnqueue<CanAlloc>(std::forward<T>(element));
211 template <
typename U>
236 size_t block_tail = front_block->localTail;
237 size_t block_front = front_block->front.load();
239 if (block_front != block_tail || block_front != (front_block->localTail = front_block->tail.load()))
243 non_empty_front_block:
245 auto element =
reinterpret_cast<T*
>(front_block->data + block_front *
sizeof(T));
246 result = std::move(*element);
249 block_front = (block_front + 1) & front_block->sizeMask;
252 front_block->front = block_front;
259 block_tail = front_block->localTail = front_block->tail.load();
260 block_front = front_block->front.load();
263 if (block_front != block_tail)
266 goto non_empty_front_block;
270 Block* next_block = front_block->next;
275 size_t next_block_front = next_block->front.load();
276 size_t next_block_tail = next_block->localTail = next_block->tail.load();
281 assert(next_block_front != next_block_tail);
290 auto element =
reinterpret_cast<T*
>(front_block->data + next_block_front *
sizeof(T));
292 result = std::move(*element);
295 next_block_front = (next_block_front + 1) & front_block->sizeMask;
298 front_block->front = next_block_front;
322 size_t block_tail = front_block->localTail;
323 size_t block_front = front_block->front.load();
325 if (block_front != block_tail || block_front != (front_block->localTail = front_block->tail.load()))
328 non_empty_front_block:
329 return reinterpret_cast<T*
>(front_block->data + block_front *
sizeof(T));
335 block_tail = front_block->localTail = front_block->tail.load();
336 block_front = front_block->front.load();
339 if (block_front != block_tail)
341 goto non_empty_front_block;
344 Block* next_block = front_block->next;
346 size_t next_block_front = next_block->front.load();
349 assert(next_block_front != next_block->tail.load());
350 return reinterpret_cast<T*
>(next_block->data + next_block_front *
sizeof(T));
367 size_t block_tail = front_block->localTail;
368 size_t block_front = front_block->front.load();
370 if (block_front != block_tail || block_front != (front_block->localTail = front_block->tail.load()))
374 non_empty_front_block:
375 auto element =
reinterpret_cast<T*
>(front_block->data + block_front *
sizeof(T));
378 block_front = (block_front + 1) & front_block->sizeMask;
381 front_block->front = block_front;
387 block_tail = front_block->localTail = front_block->tail.load();
388 block_front = front_block->front.load();
391 if (block_front != block_tail)
393 goto non_empty_front_block;
397 Block* next_block = front_block->next;
399 size_t next_block_front = next_block->front.load();
400 size_t next_block_tail = next_block->localTail = next_block->tail.load();
403 assert(next_block_front != next_block_tail);
411 auto element =
reinterpret_cast<T*
>(front_block->data + next_block_front *
sizeof(T));
414 next_block_front = (next_block_front + 1) & front_block->sizeMask;
417 front_block->front = next_block_front;
434 Block* block = front_block;
438 size_t block_front = block->front.load();
439 size_t block_tail = block->tail.load();
440 result += (block_tail - block_front) & block->sizeMask;
441 block = block->next.load();
442 }
while (block != front_block);
453 template <AllocationMode canAlloc,
typename U>
468 size_t block_front = tail_block->localFront;
469 size_t block_tail = tail_block->tail.load();
471 size_t next_block_tail = (block_tail + 1) & tail_block->sizeMask;
472 if (next_block_tail != block_front || next_block_tail != (tail_block->localFront = tail_block->front.load()))
476 char* location = tail_block->data + block_tail *
sizeof(T);
477 new (location) T(std::forward<U>(element));
480 tail_block->tail = next_block_tail;
495 Block* tail_block_next = tail_block->next.load();
496 size_t next_block_front = tail_block_next->localFront = tail_block_next->front.load();
497 next_block_tail = tail_block_next->tail.load();
502 assert(next_block_front == next_block_tail);
503 tail_block_next->localFront = next_block_front;
505 char* location = tail_block_next->data + next_block_tail *
sizeof(T);
506 new (location) T(std::forward<U>(element));
508 tail_block_next->tail = (next_block_tail + 1) & tail_block_next->sizeMask;
517 auto new_block =
makeBlock(new_block_size);
518 if (new_block ==
nullptr)
525 new (new_block->data) T(std::forward<U>(element));
527 assert(new_block->front == 0);
528 new_block->tail = new_block->localTail = 1;
530 new_block->next = tail_block->next.load();
531 tail_block->next = new_block;
549 assert(
false &&
"Should be unreachable code");
574 for (
size_t i = 1; i <
sizeof(size_t); i <<= 1)
582 template <
typename U>
585 const std::size_t alignment = std::alignment_of<U>::value;
586 return ptr + (alignment - (
reinterpret_cast<std::uintptr_t
>(ptr) % alignment)) % alignment;
595 assert(!
in_section_ &&
"ReaderWriterQueue does not support enqueuing_ or dequeuing_ elements from other " 639 Block(
size_t const& _size,
char* _rawThis,
char* _data)
646 , sizeMask(_size - 1)
662 auto size =
sizeof(Block) + std::alignment_of<Block>::value - 1;
663 size +=
sizeof(T) * capacity + std::alignment_of<T>::value - 1;
664 auto new_block_raw =
static_cast<char*
>(std::malloc(size));
665 if (new_block_raw ==
nullptr)
670 auto new_block_aligned = alignFor<Block>(new_block_raw);
671 auto new_block_data = alignFor<T>(new_block_aligned +
sizeof(Block));
672 return new (new_block_aligned) Block(capacity, new_block_raw, new_block_data);
690 template <
typename T,
size_t MAX_BLOCK_SIZE = 512>
706 if (inner_.tryEnqueue(element))
719 if (inner_.tryEnqueue(std::forward<T>(element)))
732 if (inner_.enqueue(element))
745 if (inner_.enqueue(std::forward<T>(element)))
756 template <
typename U>
761 bool success = inner_.tryDequeue(result);
771 template <
typename U>
775 bool success = inner_.tryDequeue(result);
787 template <
typename U>
790 if (!sema_.wait(timeout_usecs))
794 bool success = inner_.tryDequeue(result);
801 #if __cplusplus > 199711L || _MSC_VER >= 1700 808 template <
typename U,
typename Rep,
typename Period>
809 inline bool waitDequeTimed(U& result, std::chrono::duration<Rep, Period>
const& timeout)
811 return waitDequeTimed(result, std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
822 return inner_.peek();
832 bool result = inner_.pop();
844 return sema_.availableApprox();
AE_FORCEINLINE bool tryEnqueue(T &&element)
AE_FORCEINLINE size_t sizeApprox() const
WeakAtomic< size_t > tail
bool innerEnqueue(U &&element)
AE_FORCEINLINE T * peek()
::moodycamel::ReaderWriterQueue< T, MAX_BLOCK_SIZE > ReaderWriterQueue
WeakAtomic< size_t > front
size_t largest_block_size_
ReaderWriterQueue(size_t max_size=15)
AE_FORCEINLINE bool pop()
AE_FORCEINLINE bool enqueue(T const &element)
Block(size_t const &_size, char *_rawThis, char *_data)
static Block * makeBlock(size_t capacity)
#define MOODYCAMEL_CACHE_LINE_SIZE
BlockingReaderWriterQueue & operator=(ReaderWriterQueue const &)
static AE_FORCEINLINE size_t ceilToPow2(size_t x)
AE_FORCEINLINE bool tryEnqueue(T &&element)
ReaderWriterQueue(ReaderWriterQueue const &)
size_t sizeApprox() const
spsc_sema::LightweightSemaphore sema_
AE_FORCEINLINE bool tryEnqueue(T const &element)
BlockingReaderWriterQueue(size_t maxSize=15)
AE_FORCEINLINE bool tryEnqueue(T const &element)
AE_FORCEINLINE void compilerFence(memory_order order)
ReentrantGuard & operator=(ReentrantGuard const &)
WeakAtomic< Block * > tail_block_
static AE_FORCEINLINE char * alignFor(char *ptr)
bool tryDequeue(U &result)
BlockingReaderWriterQueue(ReaderWriterQueue const &)
WeakAtomic< Block * > front_block_
void waitDequeue(U &result)
char cachelineFiller_[MOODYCAMEL_CACHE_LINE_SIZE-sizeof(WeakAtomic< Block * >)]
WeakAtomic< Block * > next
AE_FORCEINLINE bool enqueue(T &&element)
ReentrantGuard(bool &in_section)
AE_FORCEINLINE void fence(memory_order order)
ReaderWriterQueue & operator=(ReaderWriterQueue const &)
bool waitDequeTimed(U &result, std::int64_t timeout_usecs)
bool tryDequeue(U &result)
AE_FORCEINLINE bool enqueue(T const &element)
AE_FORCEINLINE bool enqueue(T &&element)