From 60ee253fa7731e930b613ce2fd8e33112b12f67a Mon Sep 17 00:00:00 2001 From: Patrick Pelissier Date: Sat, 5 Jan 2019 13:15:50 +0100 Subject: [PATCH] Update --- doc/ISSUES.org | 149 ++++--------------------------------------------- 1 file changed, 11 insertions(+), 138 deletions(-) diff --git a/doc/ISSUES.org b/doc/ISSUES.org index 7e549122..73320511 100644 --- a/doc/ISSUES.org +++ b/doc/ISSUES.org @@ -91,151 +91,24 @@ http://htmlpreview.github.io/?https://github.com/P-p-H-d/mlib/blob/master/doc/Co Implement a lock free list. Most of the difficulty is the memory reclamation part. Typically this lock free list shall be compatible with RCU method. - A node can be in the following state: +** First step: backoff methods :DONE: +** Second step: lock free node pool : :DONE: + Done as m-c-mempool header. -- FREE: Can be used for allocation. -- IN_USE: The node is currently used in the list. -- TO_BE_RECLAIMED: The node is semantically removed, but some others may still be used it. +** Third step: Implement generic lock free list on top of it. - So separate lists shall be used for the memory handler. - It shall also be compatible with NUMA and avoid putting all nodes in the same lists if possible: - otherwise the head will become a hot spot and slow down everything. - - The ABA problem shall also be taken into account. Normaly with the provided state of the nodes, - it should not be a problem. + The ABA problem is already taken into account by the memory alloctor + provided that the lock free list doesn't try to be smart. - backoff should also be considered when using CAS. + backoff has be used when using CAS. Concurrent insertion / insertion and insertion / deletion and deletion / deletion shall be taken into account. -** First step: lock free node pool : + Questions: + - singly or doubly or dual push? + - needs to be logically deleted : needs a previous field + (NULL if not logically deleted) ? TBC -*** API based on Lock-Free Queue List - - -- node_t is { node_t *next; uint ticket; data_t } - - -- freelist_t : basic internal lockfree list used to construct more user friendly list (FIFO queue to avoid putting constraints on free & alloc at the same times + it is needed for reclaim part). - do not ensure consistensy of data. - is { _Atomic(node*) head, tail; } - - Atomic Queue List : See algorithm by Michael & Scott on Lock Free Queue List (aka MSList) - - + freelist_push (&list, &node) - + freelist_pop(&list) return node - + freelist_pop_if(&list, ticket) return node Needs minor adaptation on _pop. - - -- lf_memory_core_t : basic type responsible for the centralization of the list allocation. - + init(lf_mc_t &, int max_thread, size_t alloc_size) - + clear(lf_mc_t &) - + lc_mc_t is { - atomic_uint ticket; - atomic_uint reg_thread; // ==> genint instead - atomic_uint tab_ticket[MAX_THREAD]; - freelist_t free[MAX_THREAD]; - freelist_t to_be_reclaimed[MAX_THREAD]; - } - - -- lf_memory_thread_t: specialization for the current thread - * lf_mt_t is { int thd_idx; lf_mc_t *global;} - * init(lf_mt_t &, &lf_mc_t g) is { thd_idx := atomic_add(&g.reg_tread, 1); .global := g; } - * clear is ?? - * awake () is preparing the thread to handle operations on the list { .global.tab_ticket[.thd_idx] := atomic_add(&global.ticket, 1)+1; } - * sleep () is sleeping the thread so that it doesn't handle operations on the list and perform memory reclamation if needed { - .global.tab_ticket[.thd_idx] := UINT_MAX; - min_ticket := min(global.tab_ticket); - scan the list to_be_reclaimed while node.cpt < min_ticket, - pop the node from to_be_reclaimed and push it to the list free - lf_memory_thread_t shall be in awake state to do anything on the list. - -*** Bench - - Basic malloc / free: 9.6s - - LF Node : 10.6s - - Even if LF Node ensures a correct life cycle of the node for the read, the performance is passable. - Too much CAS: - - 51.40 6.13 6.13 51146587 0.00 0.00 lf_freelist_push - - 20.10 8.53 2.40 28143911 0.00 0.00 lf_freelist_pop - - 15.65 10.40 1.87 253579 0.01 0.02 lf_mempool_sleep - -*** Mixed API - - To overcome this limitation performs a mixed approache: - - having a simple (non atomic) list of nodes to alloc quickly the nodes. - - embedded within an atomic list of list of nodes for GC. - - Thread data will be: - - FreeList is a non atomic singly linked list. - - ToBeReclaimedList is the same - - FreeListList is an atomic queue list of non atomic singly list. - - CanBeReclaimedListList is an atomic queue list of non atomic singly list - - **** alloc() returns node: - - Pop from FreeList a node. - - If failure, atomic pop from FreeListList a list of list of free nodes. If success, update FreeList with the new list and pop a node from it. - - If failure, try an atomic from the FreeListList of other threads. If success, update FreeList with the new list and pop a node from it. - - If failure, malloc a new pool of FreeListList (TBD: all nodes in one pass? Needs a separate list to handle final free of nodes). - *** free(node) - - Push node in ToBeReclaimedList - *** GarbageCollector called on sleep: - - Compute min_ticket for all threads - - Atomic push of ToBeReclaimedList in CanBeReclaimedListList - - for each thread starting from mine: - -- Atomic pop from CanBeReclaimedListList[] if max ticket of pool of nodes < min_ticket and atomic push in FreeListList - max ticket of pool is the ticket of the top node of the list. - - ==> Should be much faster since there is much less CAS operations performed and GC is perfomed per block. - - Tests have been done: - - Performance is good. (8.9s even if most of the bench times is lost in usleep) - Real performance (without usleep) is 3x faster. - - Integrity seems ok after lot of debug. Lock Free algorithm are definitly not trivial, - even when implementing well known algorithms! - - -** Second step: lf_list_t is a stack based list based on RCU: - - _read: classic non atomic operations. - - _push(list, data, lf_mt_t) is { - allocate new node from lf_mt_t.global.free, or malloc if nothing. - copy data into node - head := atomic_load(&list.head) - do - node->next := head - while !CAS(&list.head, &head, node) - # ABA problem shall not happen since node cannot be reused immediately - - _pop(list, data, lf_mt_t) is { - head := atomic_load(&list.head) - do { - if head == NULL return NULL - } while !CAS(&list.head, &head, head.next) - - copy head.data into data - head.cpt := atomic_read(&global.ticket) - push node in global.to_be_reclaimed - } - _erase() TODO - _update() TODO - -** Example / Way to use: - - - init global memory handler. - - for each thread, - -- init thread memory handler XX, - -- ... - -- XX.awake() - -- ... - -- do as many stuff on any list as we want (this is hot spot). - -- ... - -- XX.sleep() # Memory can be reclaimed - -- XX.awake() - * TODO #15: Constructor / Destructor Prologue / Epilogue for Stack Exception Handling :ENHANCEMENT: