diff --git a/README.md b/README.md index 606b05de..46fb7756 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,6 @@ The following headers are obsolete and will be removed in the next major release * [m-concurrent.h](#m-concurrent): header for transforming a container into a concurrent container (thread safe), * [m-i-shared.h](#m-i-shared): header for creating intrusive shared pointer of generic type (Thread Safe). * [m-mempool.h](#m-mempool): header for creating specialized & fast memory allocator, -* m-c-mempool.h: WIP header for creating fast concurrent memory allocation. Each containers define their iterators (if it is meaningful). diff --git a/example/ex11-small-name.c b/example/ex11-small-name.c index f16227cd..e05b2ea2 100644 --- a/example/ex11-small-name.c +++ b/example/ex11-small-name.c @@ -8,7 +8,6 @@ #include "m-bptree.h" #include "m-bstring.h" #include "m-buffer.h" -#include "m-c-mempool.h" #include "m-concurrent.h" #include "m-deque.h" #include "m-dict.h" @@ -70,4 +69,4 @@ M_WORKER_SPAWN_DEF2(worker, (filename, m_string_t) ) int main(void) { exit(0); -} \ No newline at end of file +} diff --git a/m-c-mempool.h b/m-c-mempool.h deleted file mode 100644 index e144e496..00000000 --- a/m-c-mempool.h +++ /dev/null @@ -1,835 +0,0 @@ -/* - * M*LIB - Concurrent memory pool allocator - * - * Copyright (c) 2017-2025, Patrick Pelissier - * All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * + Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#ifndef MSTARLIB_CONCURRENT_MEMPOOL_H -#define MSTARLIB_CONCURRENT_MEMPOOL_H - -#include "m-core.h" -#include "m-atomic.h" -#include "m-genint.h" - -M_BEGIN_PROTECTED_CODE - -/* Minimum number of nodes per group of nodes */ -#define M_CMEMP00L_MIN_NODE_PER_GROUP 16 - -#define M_C_MEMPOOL_DEF(name, type_t) \ - M_BEGIN_PROTECTED_CODE \ - M_CMEMP00L_DEF_SINGLY_LIST(name, type_t) \ - M_CMEMP00L_DEF_LF_QUEUE(name, type_t) \ - M_CMEMP00L_DEF_LFMP_THREAD_MEMPOOL(name, type_t) \ - M_CMEMP00L_DEF_SYSTEM_ALLOC(name, type_t) \ - M_CMEMP00L_DEF_LF_MEMPOOL(name, type_t) \ - M_END_PROTECTED_CODE - -/* Classic internal Singly List without allocation */ -#define M_CMEMP00L_DEF_SINGLY_LIST(name, type_t) \ - \ - typedef struct M_F(name, _slist_node_s) { \ - struct M_F(name, _slist_node_s) *next; \ - type_t data; \ - } M_F(name, _slist_node_ct); \ - \ - typedef struct M_F(name, _slist_node_s) *M_F(name, _slist_ct)[1]; \ - \ - M_INLINE void \ - M_F(name, _slist_init)(M_F(name, _slist_ct) list) \ - { \ - *list = NULL; \ - } \ - \ - M_INLINE void \ - M_F(name, _slist_push)(M_F(name, _slist_ct) list, \ - M_F(name, _slist_node_ct) *node) \ - { \ - node->next = *list; \ - *list = node; \ - } \ - \ - M_INLINE M_F(name, _slist_node_ct) * \ - M_F(name, _slist_pop)(M_F(name, _slist_ct) list) \ - { \ - M_ASSERT (*list != NULL); \ - M_F(name, _slist_node_ct) *node = *list; \ - *list = node->next; \ - M_IF_DEBUG(node->next = NULL;) \ - return node; \ - } \ - \ - M_INLINE bool \ - M_F(name, _slist_empty_p)(M_F(name, _slist_ct) list) \ - { \ - return *list == NULL; \ - } \ - \ - M_INLINE void \ - M_F(name, _slist_move)(M_F(name, _slist_ct) list, \ - M_F(name, _slist_ct) src) \ - { \ - *list = *src; \ - *src = NULL; \ - } \ - \ - M_INLINE void \ - M_F(name, _slist_clear)(M_F(name, _slist_ct) list) \ - { \ - M_F(name, _slist_node_ct) *it = *list, *next; \ - while (it) { \ - next = it->next; \ - M_MEMORY_DEL(m_context, it); \ - it = next; \ - } \ - *list = NULL; \ - } \ - - -/* Lock Free free queue list (not generic one) of lists without allocation - Based on Michael & Scott Lock Free Queue List algorithm. - Each list is considered empty if there is only one node within. - This LF Queue List doesn't try to prevent the ABA problem. It is up to the - caller to avoid recycling the nodes too fast. - Each list has its own unique NIL ptr in order to avoid issues when - migrating a node from a Q to another: in the following scenario, - - Thread 1 performs a PUSH of N in Q1 with Q1 empty (only node is NA) - NA.next is NIL. - - Thread 1 is interrupted just before the CAS on NA.next - - Thread 2 performs a successful push of NB in Q1. NA.next is set to NB. - - Thread 2 performs a successful pop of NA in Q1 - - Thread 2 performs a successful push of NA in Q2. NA.next is set to NIL. - - Thread 1 is restored and will succeed as NA.next is once again NIL. - In order to prevent the last CAS to succeed, each queue uses its own NIL pointer. - It is a derived problem of the ABA problem. - */ -/* TODO: Optimize alignment to reduce memory consumption. NIL object can use [] - to reduce memory consumption too (non compatible with C++ ...) */ -#define M_CMEMP00L_DEF_LF_QUEUE(name, type_t) \ - \ - typedef struct M_F(name, _lf_node_s) { \ - M_ATTR_EXTENSION _Atomic(struct M_F(name, _lf_node_s) *) next; \ - m_gc_atomic_ticket_ct cpt; \ - M_F(name, _slist_ct) list; \ - } M_F(name, _lf_node_t); \ - \ - typedef struct M_F(name, _lflist_s) { \ - M_ATTR_EXTENSION _Atomic(M_F(name, _lf_node_t) *) head; \ - char align1[M_ALIGN_FOR_CACHELINE_EXCLUSION]; \ - M_ATTR_EXTENSION _Atomic(M_F(name, _lf_node_t) *) tail; \ - char align2[M_ALIGN_FOR_CACHELINE_EXCLUSION]; \ - M_F(name, _lf_node_t) nil; \ - } M_F(name, _lflist_ct)[1]; \ - \ - M_INLINE void \ - M_F(name, _lflist_init)(M_F(name, _lflist_ct) list, \ - M_F(name, _lf_node_t) *node) \ - { \ - atomic_init(&list->head, node); \ - atomic_init(&list->tail, node); \ - atomic_store_explicit(&node->next, &list->nil, memory_order_relaxed); \ - } \ - \ - M_INLINE bool \ - M_F(name, _lflist_empty_p)(M_F(name, _lflist_ct) list) \ - { \ - return atomic_load(&list->tail) == atomic_load(&list->head); \ - } \ - \ - M_INLINE void \ - M_F(name, _lflist_push)(M_F(name, _lflist_ct) list, \ - M_F(name, _lf_node_t) *node, m_core_backoff_ct bkoff) \ - { \ - M_F(name, _lf_node_t) *tail; \ - M_F(name, _lf_node_t) *next; \ - \ - atomic_store_explicit(&node->next, &list->nil, memory_order_relaxed); \ - m_core_backoff_reset(bkoff); \ - while (true) { \ - tail = atomic_load(&list->tail); \ - next = atomic_load_explicit(&tail->next, memory_order_acquire); \ - if (M_UNLIKELY(next != &list->nil)) { \ - /* Tail was not pointing to the last node \ - Try to swing Tail to the next node */ \ - atomic_compare_exchange_weak_explicit(&list->tail, \ - &tail, next, \ - memory_order_release, \ - memory_order_relaxed); \ - } else { \ - /* Try to link node at the end of the linked list */ \ - if (atomic_compare_exchange_strong_explicit(&tail->next, \ - &next, node, \ - memory_order_release, \ - memory_order_relaxed)) \ - break; \ - m_core_backoff_wait(bkoff); \ - } \ - } \ - /* Enqueue is done. Try to swing Tail to the inserted node \ - If it fails, someone else will do it or has already did it. */ \ - atomic_compare_exchange_strong_explicit(&list->tail, &tail, node, \ - memory_order_acq_rel, \ - memory_order_relaxed); \ - } \ - \ - M_INLINE M_F(name, _lf_node_t) * \ - M_F(name, _lflist_pop)(M_F(name, _lflist_ct) list, m_core_backoff_ct bkoff) \ - { \ - M_F(name, _lf_node_t) *head; \ - M_F(name, _lf_node_t) *tail; \ - M_F(name, _lf_node_t) *next; \ - \ - /* Reinitialize backoff */ \ - m_core_backoff_reset(bkoff); \ - while (true) { \ - head = atomic_load(&list->head); \ - tail = atomic_load(&list->tail); \ - next = atomic_load(&head->next); \ - /* Are head, tail, and next consistent?*/ \ - if (M_LIKELY(head == \ - atomic_load_explicit(&list->head, memory_order_relaxed))) \ - { \ - /* Is queue empty or Tail falling behind? */ \ - if (head == tail) { \ - /* Is queue empty? */ \ - if (next == &list->nil) \ - return NULL; \ - /* Tail is falling behind. Try to advance it */ \ - atomic_compare_exchange_strong_explicit(&list->tail, &tail, \ - next, \ - memory_order_release, \ - memory_order_relaxed); \ - } else { \ - /* Try to swing Head to the next node */ \ - if (atomic_compare_exchange_strong_explicit(&list->head, \ - &head, next, \ - memory_order_release, \ - memory_order_relaxed)) { \ - break; \ - } \ - /* Failure: perform a random exponential backoff */ \ - m_core_backoff_wait(bkoff); \ - } \ - } \ - } \ - /* dequeue returns an element that becomes the new dummy element (the new head), \ - and the former dummy element (the former head) is removed: \ - Since we want a link of free list, and we don't care about the content itself, \ - provided that the node we return is older than the one we should return, \ - Therefore, we return the previous dummy head. \ - As such, it is not the original MSqueue algorithm. */ \ - M_IF_DEBUG(atomic_store(&head->next, (M_F(name, _lf_node_t) *) 0);) \ - return head; \ - } \ - \ - /* Dequeue a node if the node is old enough */ \ - M_INLINE M_F(name, _lf_node_t) * \ - M_F(name, _lflist_pop_if)(M_F(name, _lflist_ct) list, \ - m_gc_ticket_ct age, m_core_backoff_ct bkoff) \ - { \ - M_F(name, _lf_node_t) *head; \ - M_F(name, _lf_node_t) *tail; \ - M_F(name, _lf_node_t) *next; \ - \ - m_core_backoff_reset(bkoff); \ - while (true) { \ - head = atomic_load(&list->head); \ - tail = atomic_load(&list->tail); \ - next = atomic_load(&head->next); \ - if (M_LIKELY(head == atomic_load_explicit(&list->head, memory_order_relaxed))) \ - { \ - if (head == tail) { \ - if (next == &list->nil) \ - return NULL; \ - atomic_compare_exchange_strong_explicit(&list->tail, &tail, next, \ - memory_order_release, \ - memory_order_relaxed); \ - } else { \ - /* Test if the node is old enough to be popped */ \ - if (atomic_load_explicit(&next->cpt, memory_order_relaxed) >= age) \ - return NULL; \ - /* Try to swing Head to the next node */ \ - if (atomic_compare_exchange_strong_explicit(&list->head, \ - &head, next, \ - memory_order_release, \ - memory_order_relaxed)) { \ - break; \ - } \ - m_core_backoff_wait(bkoff); \ - } \ - } \ - } \ - M_IF_DEBUG(atomic_store(&head->next, (M_F(name, _lf_node_t) *) 0);) \ - return head; \ - } \ - \ - M_INLINE void \ - M_F(name, _lflist_clear)(M_F(name, _lflist_ct) list) \ - { \ - m_core_backoff_ct bkoff; \ - m_core_backoff_init(bkoff); \ - while (true) { \ - M_F(name, _lf_node_t) *node = M_F(name, _lflist_pop)(list, bkoff); \ - if (node == NULL) break; \ - M_F(name, _lf_node_t) *next = atomic_load_explicit(&node->next, \ - memory_order_relaxed); \ - M_F(name, _slist_clear)(node->list); \ - M_MEMORY_DEL(m_context, node); \ - node = next; \ - } \ - /* Dummy node to free too */ \ - M_F(name, _lf_node_t) *dummy; \ - dummy = atomic_load_explicit(&list->head, memory_order_relaxed); \ - M_F(name, _slist_clear)(dummy->list); \ - M_MEMORY_DEL(m_context, dummy); \ - } \ - -/* System node allocator: request memory to the system. - As such it is a non Lock-Free path. */ -#define M_CMEMP00L_DEF_SYSTEM_ALLOC(name, type_t) \ - \ - M_INLINE M_F(name, _lf_node_t) * \ - M_F(name, _alloc_node)(unsigned int initial) \ - { \ - M_F(name, _lf_node_t) * node; \ - node = M_MEMORY_ALLOC(m_context, M_F(name, _lf_node_t)); \ - if (M_UNLIKELY_NOMEM (node == NULL)) { \ - M_MEMORY_FULL(M_F(name, _lf_node_t), 1); \ - } \ - atomic_init(&node->next, (M_F(name, _lf_node_t) *) 0); \ - atomic_init(&node->cpt, 0UL); \ - M_F(name, _slist_init)(node->list); \ - for(unsigned i = 0; i < initial; i++) { \ - M_F(name, _slist_node_ct) *n; \ - n = M_MEMORY_ALLOC(m_context, M_F(name, _slist_node_ct)); \ - if (M_UNLIKELY_NOMEM (n == NULL)) { \ - M_MEMORY_FULL(M_F(name, _lf_node_t), 1); \ - } \ - M_F(name, _slist_push)(node->list, n); \ - } \ - return node; \ - } \ - -/* Concurrent Memory pool - The data structure is the following. - Each thread has its own pool of nodes (local) that only it can - access (it is a singly list). If there is no longer any node in this - pool, it requests a new pool to the lock free queue of pool (group of - nodes). If it fails, it requests a new pool to the system allocator - (and from there it is no longer lock free). - This memory pool can only be lock free if the initial state is - sufficiently dimensioned to avoid calling the system allocator during - the normal processing. - Then each thread pushes its deleted node into another pool of nodes, - where the node is logically deleted (no contain of the node is destroyed - at this point and the node can be freely accessed by other threads). - Once the thread mempool is put to sleep, the age of the pool of logical - deleted nodes is computed and this pool is move to the Lock Free Queue - List of pools to be reclaimed. Then A Garbage Collector is performed - on this Lock Free Queue list to reclaim all pools that are sufficiently - aged (taking into account the grace period of the pool) to be moved back - to the Lock Free Queue of the free pools. - - Each pool of nodes can be in the following state: - * FREE state if it is present in the Lock Free Queue of free pools. - * EMPTY state if it is present in the Lock Free Queue of empty pools - which means that the nodes present in it has been USED directly by a thread, - * TO_BE_RECLAIMED state if it is present in the Lock Free Queue of TBR pools - - A pool of nodes will go to the following state: - FREE --> EMPTY --> TO_BE_RECLAIMED - ^ | - +----------------------+ - - The ABA problem is taken into account as a node cannot be reused in the - same queue without performing a full cycle of its state. Moreover - it can only move from TO_BE_RECLAIMED to FREE if and only if a grace - period is finished (and then we are sure that no thread references any - older node). - - Each thread has its own backoff structure (with local pseudo-random - generator). - - The grace period is detected through a global age counter (ticket) - that is incremented each time a thread is awaken / sleep. - Each thread has its own age that is set to the global ticket on sleep/awaken. - The age of the pool to be reclaimed is also set to this global age counter. - - To ensure that the grace period is finished, it tests if all threads - are younger than the age of the pool to be reclaimed. - - From a performance point of view, this puts a bottleneck on the global - age counter that is shared and incremented by all threads. However, - the sleep/awaken operations are much less frequent than other operations. - Thus, it shall not have a huge impact on the performance if the user - code is intelligent with the sleep/awaken operations. - - As such it won't support more than ULONG_MAX sleep for all threads. -*/ -#define M_CMEMP00L_DEF_LFMP_THREAD_MEMPOOL(name, type_t) \ - \ - typedef struct M_F(name, _lfmp_thread_s) { \ - M_F(name, _slist_ct) free; \ - M_F(name, _slist_ct) to_be_reclaimed; \ - M_CACHELINE_ALIGN(align1, M_F(name, _slist_ct), M_F(name, _slist_ct)); \ - } M_F(name, _lfmp_thread_ct); \ - \ - M_INLINE void \ - M_F(name, _lfmp_thread_init)(M_F(name, _lfmp_thread_ct) *t) \ - { \ - M_F(name, _slist_init)(t->free); \ - M_F(name, _slist_init)(t->to_be_reclaimed); \ - } \ - \ - M_INLINE void \ - M_F(name, _lfmp_thread_clear)(M_F(name, _lfmp_thread_ct) *t) \ - { \ - M_ASSERT(M_F(name, _slist_empty_p)(t->to_be_reclaimed)); \ - M_F(name, _slist_clear)(t->free); \ - M_F(name, _slist_clear)(t->to_be_reclaimed); \ - } \ - -/* NOTE: once a node is deleted, its data are kept readable until the future GC */ -#define M_CMEMP00L_DEF_LF_MEMPOOL(name, type_t) \ - \ - typedef struct M_F(name, _s) { \ - unsigned initial; \ - M_F(name, _lfmp_thread_ct) *thread_data; \ - M_F(name, _lflist_ct) free; \ - M_F(name, _lflist_ct) to_be_reclaimed; \ - M_F(name, _lflist_ct) empty; \ - m_cmemp00l_list_ct mempool_node; \ - struct m_gc_s *gc_mem; \ - } M_F(name, _t)[1]; \ - \ - /* Garbage collect of the nodes of the mempool on sleep */ \ - M_INLINE void \ - M_C3(m_cmemp00l_,name,_gc_on_sleep)(m_gc_t gc_mem, m_cmemp00l_list_ct *data, \ - m_gc_tid_t id, m_gc_ticket_ct ticket, m_gc_ticket_ct min_ticket) \ - { \ - /* Get back the mempool from the node */ \ - struct M_F(name, _s) *mempool = \ - M_TYPE_FROM_FIELD(struct M_F(name, _s), data, m_cmemp00l_list_ct, mempool_node); \ - \ - /* Move the local nodes of the mempool to be reclaimed to the thread into the global pool */ \ - if (!M_F(name, _slist_empty_p)(mempool->thread_data[id].to_be_reclaimed)) { \ - M_F(name, _lf_node_t) *node; \ - /* Get a new empty group of nodes */ \ - node = M_F(name, _lflist_pop)(mempool->empty, gc_mem->thread_data[id].bkoff); \ - if (M_UNLIKELY (node == NULL)) { \ - /* Fail to get an empty group of node. \ - Alloc a new one from the system */ \ - node = M_F(name, _alloc_node)(0); \ - M_ASSERT(node != NULL); \ - } \ - M_ASSERT(M_F(name, _slist_empty_p)(node->list)); \ - M_F(name, _slist_move)(node->list, mempool->thread_data[id].to_be_reclaimed); \ - atomic_store_explicit(&node->cpt, ticket, memory_order_relaxed); \ - M_F(name, _lflist_push)(mempool->to_be_reclaimed, node, gc_mem->thread_data[id].bkoff); \ - } \ - \ - /* Perform a GC of the freelist of nodes */ \ - while (true) { \ - M_F(name, _lf_node_t) *node; \ - node = M_F(name, _lflist_pop_if)(mempool->to_be_reclaimed, \ - min_ticket, gc_mem->thread_data[id].bkoff); \ - if (node == NULL) break; \ - M_F(name, _lflist_push)(mempool->free, node, gc_mem->thread_data[id].bkoff); \ - } \ - } \ - \ - M_INLINE void \ - M_F(name, _init)(M_F(name, _t) mem, m_gc_t gc_mem, \ - unsigned init_node_count, unsigned init_group_count) \ - { \ - const size_t max_thread = gc_mem->max_thread; \ - /* Initialize the thread data of the mempool */ \ - mem->thread_data = M_MEMORY_REALLOC(m_context, M_F(name, _lfmp_thread_ct), NULL, 0, max_thread); \ - if (M_UNLIKELY_NOMEM (mem->thread_data == NULL)) { \ - M_MEMORY_FULL(M_F(name, _lfmp_thread_ct), max_thread); \ - } \ - for(unsigned i = 0; i < max_thread;i++) { \ - M_F(name, _lfmp_thread_init)(&mem->thread_data[i]); \ - } \ - /* Preallocate some group of nodes for the mempool */ \ - mem->initial = M_MAX(M_CMEMP00L_MIN_NODE_PER_GROUP, init_node_count); \ - M_F(name, _lflist_init)(mem->free, M_F(name, _alloc_node)(init_node_count)); \ - M_F(name, _lflist_init)(mem->to_be_reclaimed, M_F(name, _alloc_node)(init_node_count)); \ - M_F(name, _lflist_init)(mem->empty, M_F(name, _alloc_node)(0)); \ - for(unsigned i = 1; i < init_group_count; i++) { \ - M_F(name, _lflist_push)(mem->free, M_F(name, _alloc_node)(init_node_count), \ - gc_mem->thread_data[0].bkoff); \ - M_F(name, _lflist_push)(mem->empty, M_F(name, _alloc_node)(0), \ - gc_mem->thread_data[0].bkoff); \ - } \ - /* Register the mempool in the GC */ \ - mem->mempool_node.gc_on_sleep = M_C3(m_cmemp00l_,name,_gc_on_sleep); \ - mem->mempool_node.next = gc_mem->mempool_list; \ - gc_mem->mempool_list = &mem->mempool_node; \ - mem->gc_mem = gc_mem; \ - } \ - \ - M_INLINE void \ - M_F(name, _clear)(M_F(name, _t) mem) \ - { \ - const unsigned max_thread = mem->gc_mem->max_thread; \ - for(unsigned i = 0; i < max_thread;i++) { \ - M_F(name, _lfmp_thread_clear)(&mem->thread_data[i]); \ - } \ - M_MEMORY_FREE(m_context, M_F(name, _lfmp_thread_ct), mem->thread_data, max_thread); \ - mem->thread_data = NULL; \ - M_F(name, _lflist_clear)(mem->empty); \ - M_F(name, _lflist_clear)(mem->free); \ - M_ASSERT(M_F(name, _lflist_empty_p)(mem->to_be_reclaimed)); \ - M_F(name, _lflist_clear)(mem->to_be_reclaimed); \ - /* TODO: Unregister from the GC? */ \ - } \ - \ - M_INLINE type_t * \ - M_F(name, _new)(M_F(name, _t) mem, m_gc_tid_t id) \ - { \ - M_F(name, _slist_node_ct) *snode; \ - M_F(name, _lf_node_t) *node; \ - while (true) { \ - /* Fast & likely path where we access the thread pool of nodes */ \ - if (M_LIKELY(!M_F(name, _slist_empty_p)(mem->thread_data[id].free))) { \ - snode = M_F(name, _slist_pop)(mem->thread_data[id].free); \ - return &snode->data; \ - } \ - /* Request a group node to the freelist of groups */ \ - node = M_F(name, _lflist_pop)(mem->free, mem->gc_mem->thread_data[id].bkoff); \ - if (M_UNLIKELY (node == NULL)) { \ - /* Request a new group to the system. Non Lock Free path */ \ - M_ASSERT(mem->initial > 0); \ - node = M_F(name, _alloc_node)(mem->initial); \ - M_ASSERT(node != NULL); \ - M_ASSERT(!M_F(name, _slist_empty_p)(node->list)); \ - } \ - M_F(name, _slist_move)(mem->thread_data[id].free, node->list); \ - /* Push back the empty group */ \ - M_ASSERT (M_F(name, _slist_empty_p)(node->list)); \ - M_F(name, _lflist_push)(mem->empty, node, mem->gc_mem->thread_data[id].bkoff); \ - } \ - } \ - \ - M_INLINE void \ - M_F(name, _del)(M_F(name, _t) mem, type_t *d, m_gc_tid_t id) \ - { \ - M_F(name, _slist_node_ct) *snode; \ - M_ASSERT( d != NULL); \ - snode = M_TYPE_FROM_FIELD(M_F(name, _slist_node_ct), d, type_t, data); \ - M_F(name, _slist_push)(mem->thread_data[id].to_be_reclaimed, snode); \ - } \ - - -/***********************************************************************/ - -/* Define the ID of a thread */ -typedef unsigned int m_gc_tid_t; - -/* Define the age of a node */ -/* TODO: Compute if sufficient (worst cast ULONG_MAX is 32 bits) */ -typedef unsigned long m_gc_ticket_ct; -typedef atomic_ulong m_gc_atomic_ticket_ct; - -/* Define the Linked List of mempools that are registered in the GC */ -struct m_gc_s; -typedef struct m_cmemp00l_list_s { - struct m_cmemp00l_list_s *next; - void (*gc_on_sleep)(struct m_gc_s *gc_mem, - struct m_cmemp00l_list_s *data, m_gc_tid_t id, - m_gc_ticket_ct ticket, m_gc_ticket_ct min_ticket); - void *data; -} m_cmemp00l_list_ct; - -/* Define the Garbage collector thread data */ -typedef struct m_gc_lfmp_thread_s { - m_gc_atomic_ticket_ct ticket; - m_core_backoff_ct bkoff; - M_CACHELINE_ALIGN(align1, atomic_ulong, m_core_backoff_ct); -} m_gc_lfmp_thread_ct; - -/* Define the Garbage collector coordinator */ -typedef struct m_gc_s { - m_gc_atomic_ticket_ct ticket; - m_gc_tid_t max_thread; - m_genint_t thread_alloc; - m_gc_lfmp_thread_ct *thread_data; - m_cmemp00l_list_ct *mempool_list; -} m_gc_t[1]; - -M_INLINE void -m_gc_init(m_gc_t gc_mem, size_t max_thread) -{ - M_ASSERT(gc_mem != NULL); - M_ASSERT(max_thread > 0 && max_thread < INT_MAX); - - atomic_init(&gc_mem->ticket, 0UL); - m_genint_init(gc_mem->thread_alloc, (unsigned int) max_thread); - gc_mem->thread_data = M_MEMORY_REALLOC(m_context, m_gc_lfmp_thread_ct, NULL, 0, max_thread); - if (M_UNLIKELY_NOMEM (gc_mem->thread_data == NULL)) { - M_MEMORY_FULL(m_gc_lfmp_thread_ct, max_thread); - } - for(unsigned i = 0; i < max_thread;i++) { - atomic_init(&gc_mem->thread_data[i].ticket, ULONG_MAX); - m_core_backoff_init(gc_mem->thread_data[i].bkoff); - } - gc_mem->max_thread = (unsigned int) max_thread; - gc_mem->mempool_list = NULL; -} - -M_INLINE void -m_gc_clear(m_gc_t gc_mem) -{ - M_ASSERT(gc_mem != NULL && gc_mem->max_thread > 0); - - for(m_gc_tid_t i = 0; i < gc_mem->max_thread;i++) { - m_core_backoff_clear(gc_mem->thread_data[i].bkoff); - } - M_MEMORY_FREE(m_context, m_gc_lfmp_thread_ct, gc_mem->thread_data, gc_mem->max_thread); - gc_mem->thread_data = NULL; - m_genint_clear(gc_mem->thread_alloc); -} - -M_INLINE m_gc_tid_t -m_gc_attach_thread(m_gc_t gc_mem) -{ - M_ASSERT(gc_mem != NULL && gc_mem->max_thread > 0); - - unsigned id = m_genint_pop(gc_mem->thread_alloc); - return M_ASSIGN_CAST(m_gc_tid_t, id); -} - -M_INLINE void -m_gc_detach_thread(m_gc_t gc_mem, m_gc_tid_t id) -{ - M_ASSERT(gc_mem != NULL && gc_mem->max_thread > 0); - M_ASSERT(id < gc_mem->max_thread); - M_ASSERT(atomic_load(&gc_mem->thread_data[id].ticket) == ULONG_MAX); - - m_genint_push(gc_mem->thread_alloc, id); -} - -M_INLINE void -m_gc_awake(m_gc_t gc_mem, m_gc_tid_t id) -{ - M_ASSERT(gc_mem != NULL && gc_mem->max_thread > 0); - M_ASSERT(id < gc_mem->max_thread); - M_ASSERT(atomic_load(&gc_mem->thread_data[id].ticket) == ULONG_MAX); - - m_gc_ticket_ct t = atomic_fetch_add(&gc_mem->ticket, 1UL) + 1; - atomic_store(&gc_mem->thread_data[id].ticket, t); -} - -M_INLINE m_gc_ticket_ct -m_cmemp00l_gc_min_ticket(m_gc_t gc_mem) -{ - m_gc_ticket_ct min = atomic_load(&gc_mem->thread_data[0].ticket); - for(m_gc_tid_t i = 1; i < gc_mem->max_thread; i++) { - m_gc_ticket_ct t = atomic_load(&gc_mem->thread_data[i].ticket); - min = M_MIN(t, min); - } - return min; -} - -M_INLINE void -m_gc_sleep(m_gc_t gc_mem, m_gc_tid_t id) -{ - /* Increase life time of the thread */ - m_gc_ticket_ct t = atomic_fetch_add(&gc_mem->ticket, 1UL); - atomic_store(&gc_mem->thread_data[id].ticket, t+1); - const m_gc_ticket_ct min_ticket = m_cmemp00l_gc_min_ticket(gc_mem); - /* Iterate over all registered mempools */ - m_cmemp00l_list_ct *it = gc_mem->mempool_list; - - while (it) { - /* Perform a garbage collect of the mempool */ - it->gc_on_sleep(gc_mem, it, id, t, min_ticket); - /* Next mempool to scan for GC */ - it = it->next; - } - /* Sleep the thread */ - atomic_store(&gc_mem->thread_data[id].ticket, ULONG_MAX); -} - - -/***********************************************************************/ -/* */ -/* Variable Length Array MEMPOOL */ -/* */ -/***********************************************************************/ - -M_CMEMP00L_DEF_SINGLY_LIST(m_vlapool, char) -M_CMEMP00L_DEF_LF_QUEUE(m_vlapool, char) -M_CMEMP00L_DEF_SYSTEM_ALLOC(m_vlapool, char) - -typedef struct m_vlapool_lfmp_thread_s { - m_vlapool_slist_ct to_be_reclaimed; - M_CACHELINE_ALIGN(align1, m_vlapool_slist_ct); -} m_vlapool_lfmp_thread_ct; - -M_INLINE void -m_vlapool_lfmp_thread_init(m_vlapool_lfmp_thread_ct *t) -{ - m_vlapool_slist_init(t->to_be_reclaimed); -} - -M_INLINE void -m_vlapool_lfmp_thread_clear(m_vlapool_lfmp_thread_ct *t) -{ - M_ASSERT(m_vlapool_slist_empty_p(t->to_be_reclaimed)); - m_vlapool_slist_clear(t->to_be_reclaimed); -} - -typedef struct m_vlapool_s { - m_vlapool_lflist_ct to_be_reclaimed; - m_vlapool_lflist_ct empty; - m_vlapool_lfmp_thread_ct *thread_data; - m_cmemp00l_list_ct mvla_node; - struct m_gc_s *gc_mem; -} m_vlapool_t[1]; - -/* Garbage collect of the nodes of the vla mempool on sleep */ -M_INLINE void -m_cmemp00l_vlapool_on_sleep(m_gc_t gc_mem, m_cmemp00l_list_ct *data, - m_gc_tid_t id, m_gc_ticket_ct ticket, m_gc_ticket_ct min_ticket) -{ - /* Get back the mempool from the node */ - struct m_vlapool_s *vlapool = - M_TYPE_FROM_FIELD(struct m_vlapool_s, data, m_cmemp00l_list_ct, mvla_node); - - /* Move the local nodes of the vlapool to be reclaimed to the thread into the global pool */ - if (!m_vlapool_slist_empty_p(vlapool->thread_data[id].to_be_reclaimed)) { - m_vlapool_lf_node_t *node; - /* Get a new empty group of nodes */ - node = m_vlapool_lflist_pop(vlapool->empty, gc_mem->thread_data[id].bkoff); - if (M_UNLIKELY (node == NULL)) { - /* Fail to get an empty group of node. - Alloc a new one from the system */ - node = m_vlapool_alloc_node(0); - M_ASSERT(node != NULL); - } - M_ASSERT(m_vlapool_slist_empty_p(node->list)); - m_vlapool_slist_move(node->list, vlapool->thread_data[id].to_be_reclaimed); - atomic_store_explicit(&node->cpt, ticket, memory_order_relaxed); - m_vlapool_lflist_push(vlapool->to_be_reclaimed, node, gc_mem->thread_data[id].bkoff); - } - - /* Perform a GC of the freelist of nodes */ - while (true) { - m_vlapool_lf_node_t *node; - node = m_vlapool_lflist_pop_if(vlapool->to_be_reclaimed, - min_ticket, gc_mem->thread_data[id].bkoff); - if (node == NULL) break; - // No reuse of VLA nodes. Free physically the node back to the system - m_vlapool_slist_clear(node->list); - // Add back the empty group of nodes - m_vlapool_slist_init(node->list); - m_vlapool_lflist_push(vlapool->empty, node, gc_mem->thread_data[id].bkoff); - } -} - -M_INLINE void -m_vlapool_init(m_vlapool_t mem, m_gc_t gc_mem) -{ - const size_t max_thread = gc_mem->max_thread; - - /* Initialize the thread data of the vlapool */ - mem->thread_data = M_MEMORY_REALLOC(m_context, m_vlapool_lfmp_thread_ct, NULL, 0, max_thread); - if (M_UNLIKELY_NOMEM (mem->thread_data == NULL)) { - M_MEMORY_FULL(m_vlapool_lfmp_thread_ct, max_thread); - } - for(unsigned i = 0; i < max_thread;i++) { - m_vlapool_lfmp_thread_init(&mem->thread_data[i]); - } - - /* Initialize the lists */ - m_vlapool_lflist_init(mem->to_be_reclaimed, m_vlapool_alloc_node(0)); - m_vlapool_lflist_init(mem->empty, m_vlapool_alloc_node(0)); - - /* Register the mempool in the GC */ - mem->mvla_node.gc_on_sleep = m_cmemp00l_vlapool_on_sleep; - mem->mvla_node.next = gc_mem->mempool_list; - gc_mem->mempool_list = &mem->mvla_node; - mem->gc_mem = gc_mem; -} - -M_INLINE void -m_vlapool_clear(m_vlapool_t mem) -{ - const unsigned max_thread = mem->gc_mem->max_thread; - for(unsigned i = 0; i < max_thread;i++) { - m_vlapool_lfmp_thread_clear(&mem->thread_data[i]); - } - M_MEMORY_FREE(m_context, m_vlapool_lfmp_thread_ct, mem->thread_data, max_thread); - mem->thread_data = NULL; - m_vlapool_lflist_clear(mem->empty); - M_ASSERT(m_vlapool_lflist_empty_p(mem->to_be_reclaimed)); - m_vlapool_lflist_clear(mem->to_be_reclaimed); - /* TODO: Unregister from the GC? */ -} - -M_INLINE void * -m_vlapool_new(m_vlapool_t mem, m_gc_tid_t id, size_t size) -{ - M_ASSERT(mem != NULL && mem->gc_mem != NULL); - M_ASSERT(id < mem->gc_mem->max_thread); - M_ASSERT( atomic_load(&mem->gc_mem->thread_data[id].ticket) != ULONG_MAX); - - // Nothing to do with theses parameters yet - (void) mem; - (void) id; - - // Ensure the size is big enough to also represent a node - size += offsetof(struct m_vlapool_slist_node_s, data); - - // Simply wrap around a system call to get the memory - char *ptr = M_MEMORY_REALLOC(m_context, char, NULL, 0, size); - // TODO: save the allocated size somewhere (create a new field in vla) - // FIXME: Bug, we call DEL operator to free the node, but we have allocated it with the REALLOC one! - // And they are not necessarily compatible as per the spec (even if they are if they both use 'free') - // And the generic used slist expects to use the DEL one, since the vlapool abuse the memory system. - // Honnestly, it is fundamentally broken and it needs a huge rewrite. - return M_UNLIKELY (ptr == NULL) ? NULL : M_ASSIGN_CAST(void *, ptr + offsetof(struct m_vlapool_slist_node_s, data)); -} - -M_INLINE void -m_vlapool_del(m_vlapool_t mem, void *d, m_gc_tid_t id) -{ - M_ASSERT(mem != NULL && mem->gc_mem != NULL); - M_ASSERT(id < mem->gc_mem->max_thread); - M_ASSERT(atomic_load(&mem->gc_mem->thread_data[id].ticket) != ULONG_MAX); - M_ASSERT(d != NULL); - - // Get back the pointer to a struct m_vlapool_slist_node_s. - d = M_ASSIGN_CAST(void *, M_ASSIGN_CAST(char *, d) - offsetof(struct m_vlapool_slist_node_s, data)); - m_vlapool_slist_node_ct *snode = M_ASSIGN_CAST(m_vlapool_slist_node_ct *, d); - // Push the logically free memory into the list of the nodes to be reclaimed. - m_vlapool_slist_push(mem->thread_data[id].to_be_reclaimed, snode); -} - -M_END_PROTECTED_CODE - -#if M_USE_SMALL_NAME -#define C_MEMPOOL_DEF M_C_MEMPOOL_DEF -#endif - -#endif diff --git a/tests/test-mcmempool.c b/tests/test-mcmempool.c deleted file mode 100644 index 79593030..00000000 --- a/tests/test-mcmempool.c +++ /dev/null @@ -1,208 +0,0 @@ -/* - * M*LIB - Test for Concurrent memory pool allocator - * - * Copyright (c) 2017-2025, Patrick Pelissier - * All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * + Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "m-c-mempool.h" -#include "m-thread.h" - -C_MEMPOOL_DEF(lf_mempool, int) - -m_gc_t gc; - -/**********************************************************************/ - -#define MAX_VALUE 100 -#define MAX_THREAD 4 - -lf_mempool_t g; - -typedef struct { - int step; - int *ptr[MAX_VALUE]; - int value[MAX_VALUE]; -} p_t; - -p_t g_tab[MAX_THREAD]; - -static void do_stuff(m_gc_tid_t id) -{ - // Mix of local and more global allocation - int *d[MAX_VALUE]; - for(int i = 0; i < MAX_VALUE; i++) { - d[i] = lf_mempool_new(g, id); - *d[i] = i*i; - } - switch (g_tab[id].step) - { - case 0: - for(int i = 0; i < 50; i++) { - g_tab[id].value[i] += i; - g_tab[id].ptr[i] = lf_mempool_new(g, id); - *g_tab[id].ptr[i] = g_tab[id].value[i]; - } - g_tab[id].step = 1; - break; - case 1: - for(int i = 50; i < MAX_VALUE; i++) { - g_tab[id].value[i] += i; - g_tab[id].ptr[i] = lf_mempool_new(g, id); - *g_tab[id].ptr[i] = g_tab[id].value[i]; - } - g_tab[id].step = 2; - break; - case 2: - for(int i = 0; i < 50; i++) { - assert(g_tab[id].ptr[i] != NULL); - assert(*g_tab[id].ptr[i] == g_tab[id].value[i] ); - lf_mempool_del(g, g_tab[id].ptr[i], id); - g_tab[id].ptr[i] = NULL; - } - g_tab[id].step = 3; - break; - case 3: - for(int i = 50; i < MAX_VALUE; i++) { - assert(g_tab[id].ptr[i] != NULL); - assert(*g_tab[id].ptr[i] == g_tab[id].value[i] ); - lf_mempool_del(g, g_tab[id].ptr[i], id); - g_tab[id].ptr[i] = NULL; - } - g_tab[id].step = 0; - break; - default: - break; - } - for(int i = 0; i < MAX_VALUE; i++) { - assert (*d[i] == i*i); - lf_mempool_del(g, d[i], id); - } -} - -static void thread1(void *arg) -{ - (void)arg; - /* Attach this thread to the GC */ - m_gc_tid_t id = m_gc_attach_thread(gc); - for(int n = 0; n < 100000; n ++) { - /* Awake the thread: now we can alloc nodes */ - m_gc_awake(gc, id); - /* Do some stuff */ - do_stuff(id); - /* Sleep the thread. A Garbage Collect may occur if possible */ - m_gc_sleep(gc, id); - } - /* Detach the thread from the GC */ - m_gc_detach_thread(gc, id); -} - -static void thread2(void *arg) -{ - (void)arg; - m_gc_tid_t id = m_gc_attach_thread(gc); - for(int n = 0; n < 10000; n ++) { - m_gc_awake(gc, id); - do_stuff(id); - m_thread_sleep(1); - m_gc_sleep(gc, id); - } - m_gc_detach_thread(gc, id); -} - -static void test(void) -{ - /* First create the Garbage Collector */ - m_gc_init (gc, MAX_THREAD); - /* Create a mempool and attach it to the Garbage Collector */ - lf_mempool_init(g, gc, 256, MAX_THREAD); - - m_thread_t id[MAX_THREAD]; - for(int i = 0; i < MAX_THREAD; i+=2) { - m_thread_create(id[i], thread1, NULL); - m_thread_create(id[i+1], thread2, NULL); - } - for(int i = 0; i < MAX_THREAD; i++) { - m_thread_join(id[i]); - } - /* Destroy the mempool */ - lf_mempool_clear(g); - /* Destroy the Garbage Collector */ - m_gc_clear (gc); -} - -static void test2(void) -{ - /* First create the Garbage Collector */ - m_gc_init (gc, MAX_THREAD); - /* Create a mempool and attach it to the Garbage Collector */ - lf_mempool_init(g, gc, 256, MAX_THREAD); - /* Create another mempool and attach it to the Garbage Collector */ - lf_mempool_t g2; - lf_mempool_init(g2, gc, 256, MAX_THREAD); - /* Register VLA */ - m_vlapool_t vla; - m_vlapool_init(vla, gc); - - /* Attach the thread */ - m_gc_tid_t id = m_gc_attach_thread(gc); - /* Awake the thread: now we can alloc nodes */ - m_gc_awake(gc, id); - - /* Test some allocation */ - void *ptr = m_vlapool_new(vla, id, sizeof (int)); - *(int*)ptr = 42; - m_vlapool_del(vla, ptr, id); - - int *pi = lf_mempool_new(g, id); - *pi = 44; - lf_mempool_del(g, pi, id); - - int *pi2 = lf_mempool_new(g2, id); - *pi2 = 46; - lf_mempool_del(g2, pi2, id); - - /* Test that the data remain accessible - as it has only be logically deleted */ - assert (*(int*)ptr == 42); - assert (*pi == 44); - assert (*pi2 == 46); - - /* Sleep the thread */ - m_gc_sleep(gc, id); - - /* Detach the thread */ - m_gc_detach_thread(gc, id); - - /* Clear everything */ - m_vlapool_clear(vla); - lf_mempool_clear(g); - lf_mempool_clear(g2); - m_gc_clear(gc); -} - -int main(void) -{ - test(); - test2(); - exit(0); -} -