mirrors/bird
mirrors/bird
0
0
Fork 0
mirror of https://gitlab.nic.cz/labs/bird.git synced 2024-11-09 20:58:44 +00:00
Code Releases Activity

RCU read lock optimization

Browse Source
This commit is contained in:
Maria Matejka 2024-06-20 22:57:52 +02:00
parent d23db54da0
commit 6f981969bb
1 changed files with 12 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/rcu.h
View File

@ -23,6 +23,7 @@ extern _Atomic u64 rcu_global_phase;
struct rcu_thread { struct rcu_thread {
struct rcu_thread * _Atomic next; struct rcu_thread * _Atomic next;
u64 local_ctl;
_Atomic u64 ctl; _Atomic u64 ctl;
}; };
@ -32,35 +33,31 @@ extern _Thread_local uint rcu_blocked;
static inline void rcu_read_lock(void) static inline void rcu_read_lock(void)
{ {
/* Increment the nesting counter */ /* Increment the nesting counter */
u64 before = atomic_fetch_add_explicit( atomic_store_explicit(&this_rcu_thread.ctl, (this_rcu_thread.local_ctl += RCU_NEST_CNT), memory_order_release);
&this_rcu_thread.ctl,
RCU_NEST_CNT,
memory_order_acq_rel
);
if (before & RCU_NEST_MASK) /* Just nested */
u64 local_nest = this_rcu_thread.local_ctl & RCU_NEST_MASK;
if (local_nest > RCU_NEST_CNT)
return; return;
/* Update the phase */ ASSUME(local_nest == RCU_NEST_CNT);
u64 phase = atomic_load_explicit(&rcu_global_phase, memory_order_acquire);
u64 dif = (before & ~RCU_NEST_MASK) ^ phase;
if (dif) /* Update the phase */
atomic_fetch_xor_explicit( u64 new = atomic_load_explicit(&rcu_global_phase, memory_order_acquire) + RCU_NEST_CNT;
&this_rcu_thread.ctl, atomic_store_explicit(&this_rcu_thread.ctl, new, memory_order_release);
dif, this_rcu_thread.local_ctl = new;
memory_order_acq_rel);
} }
static inline void rcu_read_unlock(void) static inline void rcu_read_unlock(void)
{ {
/* Just decrement the nesting counter; when unlocked, nobody cares */ /* Just decrement the nesting counter; when unlocked, nobody cares */
atomic_fetch_sub_explicit(&this_rcu_thread.ctl, RCU_NEST_CNT, memory_order_acq_rel); atomic_fetch_sub_explicit(&this_rcu_thread.ctl, RCU_NEST_CNT, memory_order_acq_rel);
this_rcu_thread.local_ctl--;
} }
static inline _Bool rcu_read_active(void) static inline _Bool rcu_read_active(void)
{ {
return !!(atomic_load_explicit(&this_rcu_thread.ctl, memory_order_acquire) & RCU_NEST_MASK); return !!(this_rcu_thread.local_ctl & RCU_NEST_MASK);
} }
void synchronize_rcu(void); void synchronize_rcu(void);