diff --git a/aclocal.m4 b/aclocal.m4 index 58c48791..cc45b54a 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1,5 +1,6 @@ dnl ** Additional Autoconf tests for BIRD configure script dnl ** (c) 1999 Martin Mares +dnl ** (c) 2021 Maria Matejka AC_DEFUN([BIRD_CHECK_POINTER_ALIGNMENT], [ @@ -35,14 +36,23 @@ AC_DEFUN([BIRD_CHECK_THREAD_LOCAL], AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [ - _Thread_local static int x = 42; + static _Thread_local int x = 42; ], [] ) ], [bird_cv_thread_local=yes], + [AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM( + [ + static __thread int x = 42; + ], + [] + ) + ], + [bird_cv_thread_local=__thread], [bird_cv_thread_local=no] - ) + )]) ) ]) diff --git a/configure.ac b/configure.ac index 321bed95..5e5f2c4e 100644 --- a/configure.ac +++ b/configure.ac @@ -36,12 +36,6 @@ AC_ARG_ENABLE([memcheck], [enable_memcheck=yes] ) -AC_ARG_ENABLE([pthreads], - [AS_HELP_STRING([--enable-pthreads], [enable POSIX threads support @<:@try@:>@])], - [], - [enable_pthreads=try] -) - AC_ARG_ENABLE([libssh], [AS_HELP_STRING([--enable-libssh], [enable LibSSH support in RPKI @<:@try@:>@])], [], @@ -125,25 +119,19 @@ if test -z "$GCC" ; then fi BIRD_CHECK_THREAD_LOCAL -if test "$bird_cv_thread_local" = yes ; then - AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define to 1 if _Thread_local is available]) +if test "$bird_cv_thread_local" = no ; then + AC_MSG_ERROR([This program requires thread local storage.]) +elif test "$bird_cv_thread_local" != yes ; then + AC_DEFINE_UNQUOTED([_Thread_local], [$bird_cv_thread_local], [Legacy _Thread_local]) fi -if test "$enable_pthreads" != no ; then - BIRD_CHECK_PTHREADS +BIRD_CHECK_PTHREADS - if test "$bird_cv_lib_pthreads" = yes ; then - AC_DEFINE([USE_PTHREADS], [1], [Define to 1 if pthreads are enabled]) - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - proto_bfd=bfd - elif test "$enable_pthreads" = yes ; then - AC_MSG_ERROR([POSIX threads not available.]) - fi - - if test "$enable_pthreads" = try ; then - enable_pthreads="$bird_cv_lib_pthreads" - fi +if test "$bird_cv_lib_pthreads" = yes ; then + CFLAGS="$CFLAGS -pthread" + LDFLAGS="$LDFLAGS -pthread" +else + AC_MSG_ERROR([POSIX threads not available.]) fi # This is assumed to be necessary for proper BIRD build @@ -304,8 +292,7 @@ if test "$enable_mpls_kernel" != no ; then fi fi -all_protocols="$proto_bfd babel bgp mrt ospf perf pipe radv rip rpki static" - +all_protocols="bfd babel bgp mrt ospf perf pipe radv rip rpki static" all_protocols=`echo $all_protocols | sed 's/ /,/g'` if test "$with_protocols" = all ; then @@ -351,9 +338,15 @@ case $sysdesc in esac AC_CHECK_HEADERS_ONCE([alloca.h syslog.h]) -AC_CHECK_HEADER([sys/mman.h], [AC_DEFINE([HAVE_MMAP], [1], [Define to 1 if mmap() is available.])]) +AC_CHECK_HEADER([sys/mman.h], [AC_DEFINE([HAVE_MMAP], [1], [Define to 1 if mmap() is available.])], have_mman=no) +AC_CHECK_FUNC([aligned_alloc], [AC_DEFINE([HAVE_ALIGNED_ALLOC], [1], [Define to 1 if aligned_alloc() is available.])], have_aligned_alloc=no) AC_CHECK_MEMBERS([struct sockaddr.sa_len], [], [], [#include ]) +if test "$have_aligned_alloc" = "no" && test "$have_mman" = "no" ; then + AC_MSG_ERROR([No means of aligned alloc found. Need mmap() or aligned_alloc().]) +fi + + AC_C_BIGENDIAN( [AC_DEFINE([CPU_BIG_ENDIAN], [1], [Define to 1 if cpu is big endian])], [AC_DEFINE([CPU_LITTLE_ENDIAN], [1], [Define to 1 if cpu is little endian])], @@ -409,7 +402,7 @@ if test "$enable_debug" = yes ; then fi fi - if test "enable_debug_expensive" = yes ; then + if test "$enable_debug_expensive" = yes ; then AC_DEFINE([ENABLE_EXPENSIVE_CHECKS], [1], [Define to 1 if you want to run expensive consistency checks.]) fi fi @@ -474,7 +467,6 @@ AC_MSG_RESULT([ Object directory: $objdir]) AC_MSG_RESULT([ Iproute2 directory: $iproutedir]) AC_MSG_RESULT([ System configuration: $sysdesc]) AC_MSG_RESULT([ Debugging: $enable_debug]) -AC_MSG_RESULT([ POSIX threads: $enable_pthreads]) AC_MSG_RESULT([ Routing protocols: $protocols]) AC_MSG_RESULT([ LibSSH support in RPKI: $enable_libssh]) AC_MSG_RESULT([ Kernel MPLS support: $enable_mpls_kernel]) diff --git a/lib/birdlib.h b/lib/birdlib.h index bdf69309..dbc754d4 100644 --- a/lib/birdlib.h +++ b/lib/birdlib.h @@ -93,10 +93,6 @@ static inline int u64_cmp(u64 i1, u64 i2) #define STATIC_ASSERT(EXP) _Static_assert(EXP, #EXP) #define STATIC_ASSERT_MSG(EXP,MSG) _Static_assert(EXP, MSG) -#ifndef HAVE_THREAD_LOCAL -#define _Thread_local -#endif - /* Microsecond time */ typedef s64 btime; diff --git a/lib/coro.h b/lib/coro.h new file mode 100644 index 00000000..51712b36 --- /dev/null +++ b/lib/coro.h @@ -0,0 +1,26 @@ +/* + * BIRD Coroutines + * + * (c) 2017 Martin Mares + * (c) 2020 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_CORO_H_ +#define _BIRD_CORO_H_ + +#include "lib/resource.h" + +/* A completely opaque coroutine handle. */ +struct coroutine; + +/* Coroutines are independent threads bound to pools. + * You request a coroutine by calling coro_run(). + * It is forbidden to free a running coroutine from outside. + * The running coroutine must free itself by rfree() before returning. + */ +struct coroutine *coro_run(pool *, void (*entry)(void *), void *data); + + +#endif diff --git a/lib/lists.c b/lib/lists.c index 1aa58085..8f95c7c2 100644 --- a/lib/lists.c +++ b/lib/lists.c @@ -26,7 +26,7 @@ #define _BIRD_LISTS_C_ -#include "nest/bird.h" +#include "lib/birdlib.h" #include "lib/lists.h" LIST_INLINE int diff --git a/lib/locking.h b/lib/locking.h new file mode 100644 index 00000000..eef60154 --- /dev/null +++ b/lib/locking.h @@ -0,0 +1,44 @@ +/* + * BIRD Library -- Locking + * + * (c) 2020--2021 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_LOCKING_H_ +#define _BIRD_LOCKING_H_ + +struct domain_generic; + +/* Here define the global lock order; first to last. */ +struct lock_order { + struct domain_generic *the_bird; +}; + +extern _Thread_local struct lock_order locking_stack; +extern _Thread_local struct domain_generic **last_locked; + +#define DOMAIN(type) struct domain__##type +#define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; } + +#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) } +struct domain_generic *domain_new(const char *name, uint order); + +#define DOMAIN_NULL(type) (DOMAIN(type)) {} + +#define LOCK_DOMAIN(type, d) do_lock(((d).type), &(locking_stack.type)) +#define UNLOCK_DOMAIN(type, d) do_unlock(((d).type), &(locking_stack.type)) + +/* Internal for locking */ +void do_lock(struct domain_generic *dg, struct domain_generic **lsp); +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp); + +/* Use with care. To be removed in near future. */ +DEFINE_DOMAIN(the_bird); +extern DOMAIN(the_bird) the_bird_domain; + +#define the_bird_lock() LOCK_DOMAIN(the_bird, the_bird_domain) +#define the_bird_unlock() UNLOCK_DOMAIN(the_bird, the_bird_domain) + +#endif diff --git a/lib/timer.c b/lib/timer.c index c47e0bbc..295eeaef 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -32,6 +32,7 @@ #include "nest/bird.h" +#include "lib/coro.h" #include "lib/heap.h" #include "lib/resource.h" #include "lib/timer.h" @@ -40,53 +41,16 @@ struct timeloop main_timeloop; -#ifdef USE_PTHREADS - #include /* Data accessed and modified from proto/bfd/io.c */ -pthread_key_t current_time_key; +_Thread_local struct timeloop *local_timeloop; -static inline struct timeloop * -timeloop_current(void) -{ - return pthread_getspecific(current_time_key); -} - -static inline void -timeloop_init_current(void) -{ - pthread_key_create(¤t_time_key, NULL); - pthread_setspecific(current_time_key, &main_timeloop); -} +_Atomic btime last_time; +_Atomic btime real_time; void wakeup_kick_current(void); -#else - -/* Just use main timelooop */ -static inline struct timeloop * timeloop_current(void) { return &main_timeloop; } -static inline void timeloop_init_current(void) { } - -#endif - -btime -current_time(void) -{ - return timeloop_current()->last_time; -} - -btime -current_real_time(void) -{ - struct timeloop *loop = timeloop_current(); - - if (!loop->real_time) - times_update_real_time(loop); - - return loop->real_time; -} - #define TIMER_LESS(a,b) ((a)->expires < (b)->expires) #define TIMER_SWAP(heap,a,b,t) (t = heap[a], heap[a] = heap[b], heap[b] = t, \ @@ -138,30 +102,29 @@ tm_new(pool *p) void tm_set(timer *t, btime when) { - struct timeloop *loop = timeloop_current(); - uint tc = timers_count(loop); + uint tc = timers_count(local_timeloop); if (!t->expires) { t->index = ++tc; t->expires = when; - BUFFER_PUSH(loop->timers) = t; - HEAP_INSERT(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP); + BUFFER_PUSH(local_timeloop->timers) = t; + HEAP_INSERT(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP); } else if (t->expires < when) { t->expires = when; - HEAP_INCREASE(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + HEAP_INCREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); } else if (t->expires > when) { t->expires = when; - HEAP_DECREASE(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); } #ifdef CONFIG_BFD /* Hack to notify BFD loops */ - if ((loop != &main_timeloop) && (t->index == 1)) + if ((local_timeloop != &main_timeloop) && (t->index == 1)) wakeup_kick_current(); #endif } @@ -178,11 +141,10 @@ tm_stop(timer *t) if (!t->expires) return; - struct timeloop *loop = timeloop_current(); - uint tc = timers_count(loop); + uint tc = timers_count(local_timeloop); - HEAP_DELETE(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); - BUFFER_POP(loop->timers); + HEAP_DELETE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + BUFFER_POP(local_timeloop->timers); t->index = -1; t->expires = 0; @@ -191,8 +153,6 @@ tm_stop(timer *t) void timers_init(struct timeloop *loop, pool *p) { - times_init(loop); - BUFFER_INIT(loop->timers, p, 4); BUFFER_PUSH(loop->timers) = NULL; } @@ -205,8 +165,8 @@ timers_fire(struct timeloop *loop) btime base_time; timer *t; - times_update(loop); - base_time = loop->last_time; + times_update(); + base_time = current_time(); while (t = timers_first(loop)) { @@ -217,8 +177,8 @@ timers_fire(struct timeloop *loop) { btime when = t->expires + t->recurrent; - if (when <= loop->last_time) - when = loop->last_time + t->recurrent; + if (when <= base_time) + when = base_time + t->recurrent; if (t->randomize) when += random() % (t->randomize + 1); @@ -241,7 +201,7 @@ void timer_init(void) { timers_init(&main_timeloop, &root_pool); - timeloop_init_current(); + local_timeloop = &main_timeloop; } diff --git a/lib/timer.h b/lib/timer.h index c5ea430c..b201b8c8 100644 --- a/lib/timer.h +++ b/lib/timer.h @@ -14,6 +14,10 @@ #include "lib/buffer.h" #include "lib/resource.h" +#include + +extern _Atomic btime last_time; +extern _Atomic btime real_time; typedef struct timer { @@ -31,8 +35,6 @@ typedef struct timer struct timeloop { BUFFER_(timer *) timers; - btime last_time; - btime real_time; }; static inline uint timers_count(struct timeloop *loop) @@ -42,9 +44,10 @@ static inline timer *timers_first(struct timeloop *loop) { return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } extern struct timeloop main_timeloop; +extern _Thread_local struct timeloop *local_timeloop; -btime current_time(void); -btime current_real_time(void); +#define current_time() atomic_load_explicit(&last_time, memory_order_acquire) +#define current_real_time() atomic_load_explicit(&real_time, memory_order_acquire) //#define now (current_time() TO_S) //#define now_real (current_real_time() TO_S) @@ -94,9 +97,7 @@ tm_start_max(timer *t, btime after) } /* In sysdep code */ -void times_init(struct timeloop *loop); -void times_update(struct timeloop *loop); -void times_update_real_time(struct timeloop *loop); +void times_update(void); /* For I/O loop */ void timers_init(struct timeloop *loop, pool *p); diff --git a/proto/bfd/io.c b/proto/bfd/io.c index e696cc89..2805e0f2 100644 --- a/proto/bfd/io.c +++ b/proto/bfd/io.c @@ -52,29 +52,15 @@ struct birdloop * Current thread context */ -static pthread_key_t current_loop_key; -extern pthread_key_t current_time_key; - -static inline struct birdloop * -birdloop_current(void) -{ - return pthread_getspecific(current_loop_key); -} +static _Thread_local struct birdloop *birdloop_current; static inline void birdloop_set_current(struct birdloop *loop) { - pthread_setspecific(current_loop_key, loop); - pthread_setspecific(current_time_key, loop ? &loop->time : &main_timeloop); + birdloop_current = loop; + local_timeloop = loop ? &loop->time : &main_timeloop; } -static inline void -birdloop_init_current(void) -{ - pthread_key_create(¤t_loop_key, NULL); -} - - /* * Wakeup code for birdloop */ @@ -162,10 +148,8 @@ wakeup_kick(struct birdloop *loop) void wakeup_kick_current(void) { - struct birdloop *loop = birdloop_current(); - - if (loop && loop->poll_active) - wakeup_kick(loop); + if (birdloop_current && birdloop_current->poll_active) + wakeup_kick(birdloop_current); } @@ -188,22 +172,20 @@ events_init(struct birdloop *loop) static void events_fire(struct birdloop *loop) { - times_update(&loop->time); + times_update(); ev_run_list(&loop->event_list); } void ev2_schedule(event *e) { - struct birdloop *loop = birdloop_current(); - - if (loop->poll_active && EMPTY_LIST(loop->event_list)) - wakeup_kick(loop); + if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list)) + wakeup_kick(birdloop_current); if (e->n.next) rem_node(&e->n); - add_tail(&loop->event_list, &e->n); + add_tail(&birdloop_current->event_list, &e->n); } @@ -238,9 +220,7 @@ sockets_add(struct birdloop *loop, sock *s) void sk_start(sock *s) { - struct birdloop *loop = birdloop_current(); - - sockets_add(loop, s); + sockets_add(birdloop_current, s); } static void @@ -261,14 +241,12 @@ sockets_remove(struct birdloop *loop, sock *s) void sk_stop(sock *s) { - struct birdloop *loop = birdloop_current(); + sockets_remove(birdloop_current, s); - sockets_remove(loop, s); - - if (loop->poll_active) + if (birdloop_current->poll_active) { - loop->close_scheduled = 1; - wakeup_kick(loop); + birdloop_current->close_scheduled = 1; + wakeup_kick(birdloop_current); } else close(s->fd); @@ -354,7 +332,7 @@ sockets_fire(struct birdloop *loop) sock **psk = loop->poll_sk.data; int poll_num = loop->poll_fd.used - 1; - times_update(&loop->time); + times_update(); /* Last fd is internal wakeup fd */ if (pfd[poll_num].revents & POLLIN) @@ -387,16 +365,11 @@ sockets_fire(struct birdloop *loop) * Birdloop */ -static void * birdloop_main(void *arg); +static void *birdloop_main(void *arg); struct birdloop * birdloop_new(void) { - /* FIXME: this init should be elsewhere and thread-safe */ - static int init = 0; - if (!init) - { birdloop_init_current(); init = 1; } - pool *p = rp_new(NULL, "Birdloop root"); struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); loop->pool = p; @@ -490,7 +463,7 @@ birdloop_main(void *arg) events_fire(loop); timers_fire(&loop->time); - times_update(&loop->time); + times_update(); if (events_waiting(loop)) timeout = 0; else if (t = timers_first(&loop->time)) diff --git a/sysdep/unix/Makefile b/sysdep/unix/Makefile index 51ab98a9..a01e8f46 100644 --- a/sysdep/unix/Makefile +++ b/sysdep/unix/Makefile @@ -1,4 +1,4 @@ -src := alloc.c io.c krt.c log.c main.c random.c +src := alloc.c io.c krt.c log.c main.c random.c coroutine.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c new file mode 100644 index 00000000..2eba142c --- /dev/null +++ b/sysdep/unix/coroutine.c @@ -0,0 +1,179 @@ +/* + * BIRD Coroutines + * + * (c) 2017 Martin Mares + * (c) 2020 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#undef LOCAL_DEBUG + +#undef DEBUG_LOCKING + +#include "lib/birdlib.h" +#include "lib/locking.h" +#include "lib/coro.h" +#include "lib/resource.h" +#include "lib/timer.h" + +/* Using a rather big stack for coroutines to allow for stack-local allocations. + * In real world, the kernel doesn't alloc this memory until it is used. + * */ +#define CORO_STACK_SIZE 1048576 + +/* + * Implementation of coroutines based on POSIX threads + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Locking subsystem + */ + +_Thread_local struct lock_order locking_stack = {}; +_Thread_local struct domain_generic **last_locked = NULL; + +#define ASSERT_NO_LOCK ASSERT_DIE(last_locked == NULL) + +struct domain_generic { + pthread_mutex_t mutex; + uint order; + struct domain_generic **prev; + struct lock_order *locked_by; + const char *name; +}; + +#define DOMAIN_INIT(_name, _order) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name, .order = _order } + +static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD", OFFSETOF(struct lock_order, the_bird)); + +DOMAIN(the_bird) the_bird_domain = { .the_bird = &the_bird_domain_gen }; + +struct domain_generic * +domain_new(const char *name, uint order) +{ + ASSERT_DIE(order < sizeof(struct lock_order)); + struct domain_generic *dg = xmalloc(sizeof(struct domain_generic)); + *dg = (struct domain_generic) DOMAIN_INIT(name, order); + return dg; +} + +void +domain_free(struct domain_generic *dg) +{ + pthread_mutex_destroy(&dg->mutex); + xfree(dg); +} + +void do_lock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to lock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + + if (lsp <= last_locked) + bug("Trying to lock in a bad order"); + if (*lsp) + bug("Inconsistent locking stack state on lock"); + + pthread_mutex_lock(&dg->mutex); + + if (dg->prev || dg->locked_by) + bug("Previous unlock not finished correctly"); + dg->prev = last_locked; + *lsp = dg; + last_locked = lsp; + dg->locked_by = &locking_stack; +} + +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to unlock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + + if (dg->locked_by != &locking_stack) + bug("Inconsistent domain state on unlock"); + if ((last_locked != lsp) || (*lsp != dg)) + bug("Inconsistent locking stack state on unlock"); + dg->locked_by = NULL; + last_locked = dg->prev; + *lsp = NULL; + dg->prev = NULL; + pthread_mutex_unlock(&dg->mutex); +} + +/* Coroutines */ +struct coroutine { + resource r; + pthread_t id; + pthread_attr_t attr; + void (*entry)(void *); + void *data; +}; + +static _Thread_local _Bool coro_cleaned_up = 0; + +static void coro_free(resource *r) +{ + struct coroutine *c = (void *) r; + ASSERT_DIE(pthread_equal(pthread_self(), c->id)); + pthread_attr_destroy(&c->attr); + coro_cleaned_up = 1; +} + +static struct resclass coro_class = { + .name = "Coroutine", + .size = sizeof(struct coroutine), + .free = coro_free, +}; + +static void *coro_entry(void *p) +{ + struct coroutine *c = p; + ASSERT_DIE(c->entry); + + c->entry(c->data); + ASSERT_DIE(coro_cleaned_up); + + return NULL; +} + +struct coroutine *coro_run(pool *p, void (*entry)(void *), void *data) +{ + ASSERT_DIE(entry); + ASSERT_DIE(p); + + struct coroutine *c = ralloc(p, &coro_class); + + c->entry = entry; + c->data = data; + + int e = 0; + + if (e = pthread_attr_init(&c->attr)) + die("pthread_attr_init() failed: %M", e); + + if (e = pthread_attr_setstacksize(&c->attr, CORO_STACK_SIZE)) + die("pthread_attr_setstacksize(%u) failed: %M", CORO_STACK_SIZE, e); + + if (e = pthread_attr_setdetachstate(&c->attr, PTHREAD_CREATE_DETACHED)) + die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e); + + if (e = pthread_create(&c->id, &c->attr, coro_entry, c)) + die("pthread_create() failed: %M", e); + + return c; +} diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 810e782d..a3f1491a 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -36,6 +36,7 @@ #include "lib/resource.h" #include "lib/socket.h" #include "lib/event.h" +#include "lib/locking.h" #include "lib/timer.h" #include "lib/string.h" #include "nest/iface.h" @@ -122,55 +123,50 @@ rf_fileno(struct rfile *f) btime boot_time; + void -times_init(struct timeloop *loop) +times_update(void) { struct timespec ts; int rv; + btime old_time = current_time(); + btime old_real_time = current_real_time(); + rv = clock_gettime(CLOCK_MONOTONIC, &ts); if (rv < 0) die("Monotonic clock is missing"); if ((ts.tv_sec < 0) || (((u64) ts.tv_sec) > ((u64) 1 << 40))) log(L_WARN "Monotonic clock is crazy"); - - loop->last_time = ts.tv_sec S + ts.tv_nsec NS; - loop->real_time = 0; -} - -void -times_update(struct timeloop *loop) -{ - struct timespec ts; - int rv; - - rv = clock_gettime(CLOCK_MONOTONIC, &ts); - if (rv < 0) - die("clock_gettime: %m"); - + btime new_time = ts.tv_sec S + ts.tv_nsec NS; - if (new_time < loop->last_time) + if (new_time < old_time) log(L_ERR "Monotonic clock is broken"); - loop->last_time = new_time; - loop->real_time = 0; -} - -void -times_update_real_time(struct timeloop *loop) -{ - struct timespec ts; - int rv; - rv = clock_gettime(CLOCK_REALTIME, &ts); if (rv < 0) die("clock_gettime: %m"); - loop->real_time = ts.tv_sec S + ts.tv_nsec NS; -} + btime new_real_time = ts.tv_sec S + ts.tv_nsec NS; + if (!atomic_compare_exchange_strong_explicit( + &last_time, + &old_time, + new_time, + memory_order_acq_rel, + memory_order_relaxed)) + DBG("Time update collision: last_time"); + + if (!atomic_compare_exchange_strong_explicit( + &real_time, + &old_real_time, + new_real_time, + memory_order_acq_rel, + memory_order_relaxed)) + DBG("Time update collision: real_time"); +} /** * DOC: Sockets @@ -2037,30 +2033,17 @@ struct event_log_entry static struct event_log_entry event_log[EVENT_LOG_LENGTH]; static struct event_log_entry *event_open; static int event_log_pos, event_log_num, watchdog_active; -static btime last_time; +static btime last_io_time; static btime loop_time; static void io_update_time(void) { - struct timespec ts; - int rv; - - /* - * This is third time-tracking procedure (after update_times() above and - * times_update() in BFD), dedicated to internal event log and latency - * tracking. Hopefully, we consolidate these sometimes. - */ - - rv = clock_gettime(CLOCK_MONOTONIC, &ts); - if (rv < 0) - die("clock_gettime: %m"); - - last_time = ts.tv_sec S + ts.tv_nsec NS; + last_io_time = current_time(); if (event_open) { - event_open->duration = last_time - event_open->timestamp; + event_open->duration = last_io_time - event_open->timestamp; if (event_open->duration > config->latency_limit) log(L_WARN "Event 0x%p 0x%p took %u.%03u ms", @@ -2089,7 +2072,7 @@ io_log_event(void *hook, void *data) en->hook = hook; en->data = data; - en->timestamp = last_time; + en->timestamp = last_io_time; en->duration = 0; event_log_num++; @@ -2117,14 +2100,14 @@ io_log_dump(void) struct event_log_entry *en = event_log + (event_log_pos + i) % EVENT_LOG_LENGTH; if (en->hook) log(L_DEBUG " Event 0x%p 0x%p at %8d for %d ms", en->hook, en->data, - (int) ((last_time - en->timestamp) TO_MS), (int) (en->duration TO_MS)); + (int) ((last_io_time - en->timestamp) TO_MS), (int) (en->duration TO_MS)); } } void watchdog_sigalrm(int sig UNUSED) { - /* Update last_time and duration, but skip latency check */ + /* Update last_io_time and duration, but skip latency check */ config->latency_limit = 0xffffffff; io_update_time(); @@ -2137,7 +2120,7 @@ watchdog_start1(void) { io_update_time(); - loop_time = last_time; + loop_time = last_io_time; } static inline void @@ -2145,7 +2128,7 @@ watchdog_start(void) { io_update_time(); - loop_time = last_time; + loop_time = last_io_time; event_log_num = 0; if (config->watchdog_timeout) @@ -2166,7 +2149,7 @@ watchdog_stop(void) watchdog_active = 0; } - btime duration = last_time - loop_time; + btime duration = last_io_time - loop_time; if (duration > config->watchdog_warning) log(L_WARN "I/O loop cycle took %u.%03u ms for %d events", (uint) (duration TO_MS), (uint) (duration % 1000), event_log_num); @@ -2196,6 +2179,15 @@ static int short_loops = 0; #define SHORT_LOOP_MAX 10 #define WORK_EVENTS_MAX 10 +static int poll_reload_pipe[2]; + +void +io_loop_reload(void) +{ + char b; + write(poll_reload_pipe[1], &b, 1); +} + void io_loop(void) { @@ -2207,10 +2199,13 @@ io_loop(void) int fdmax = 256; struct pollfd *pfd = xmalloc(fdmax * sizeof(struct pollfd)); + if (pipe(poll_reload_pipe) < 0) + die("pipe(poll_reload_pipe) failed: %m"); + watchdog_start1(); for(;;) { - times_update(&main_timeloop); + times_update(); events = ev_run_list(&global_event_list); events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events; timers_fire(&main_timeloop); @@ -2220,12 +2215,17 @@ io_loop(void) poll_tout = (events ? 0 : 3000); /* Time in milliseconds */ if (t = timers_first(&main_timeloop)) { - times_update(&main_timeloop); + times_update(); timeout = (tm_remains(t) TO_MS) + 1; poll_tout = MIN(poll_tout, timeout); } - nfds = 0; + /* A hack to reload main io_loop() when something has changed asynchronously. */ + pfd[0].fd = poll_reload_pipe[0]; + pfd[0].events = POLLIN; + + nfds = 1; + WALK_LIST(n, sock_list) { pfd[nfds] = (struct pollfd) { .fd = -1 }; /* everything other set to 0 by this */ @@ -2284,7 +2284,9 @@ io_loop(void) /* And finally enter poll() to find active sockets */ watchdog_stop(); + the_bird_unlock(); pout = poll(pfd, nfds, poll_tout); + the_bird_lock(); watchdog_start(); if (pout < 0) @@ -2295,7 +2297,15 @@ io_loop(void) } if (pout) { - times_update(&main_timeloop); + if (pfd[0].revents & POLLIN) + { + /* IO loop reload requested */ + char b; + read(poll_reload_pipe[0], &b, 1); + continue; + } + + times_update(); /* guaranteed to be non-empty */ current_sock = SKIP_BACK(sock, n, HEAD(sock_list)); diff --git a/sysdep/unix/log.c b/sysdep/unix/log.c index 4e9df069..f48588b6 100644 --- a/sysdep/unix/log.c +++ b/sysdep/unix/log.c @@ -15,6 +15,7 @@ * user's manual. */ +#include #include #include #include @@ -35,8 +36,10 @@ static FILE *dbgf; static list *current_log_list; static char *current_syslog_name; /* NULL -> syslog closed */ +static _Atomic uint max_coro_id = ATOMIC_VAR_INIT(1); +static _Thread_local uint this_coro_id; -#ifdef USE_PTHREADS +#define THIS_CORO_ID (this_coro_id ?: (this_coro_id = atomic_fetch_add_explicit(&max_coro_id, 1, memory_order_acq_rel))) #include @@ -48,15 +51,6 @@ static pthread_t main_thread; void main_thread_init(void) { main_thread = pthread_self(); } static int main_thread_self(void) { return pthread_equal(pthread_self(), main_thread); } -#else - -static inline void log_lock(void) { } -static inline void log_unlock(void) { } -void main_thread_init(void) { } -static int main_thread_self(void) { return 1; } - -#endif - #ifdef HAVE_SYSLOG_H #include @@ -189,7 +183,7 @@ log_commit(int class, buffer *buf) l->pos += msg_len; } - fprintf(l->fh, "%s <%s> ", tbuf, class_names[class]); + fprintf(l->fh, "%s [%04x] <%s> ", tbuf, THIS_CORO_ID, class_names[class]); } fputs(buf->start, l->fh); fputc('\n', l->fh); @@ -299,6 +293,8 @@ die(const char *msg, ...) exit(1); } +static struct timespec dbg_time_start; + /** * debug - write to debug output * @msg: a printf-like message @@ -311,12 +307,33 @@ debug(const char *msg, ...) { #define MAX_DEBUG_BUFSIZE 16384 va_list args; - char buf[MAX_DEBUG_BUFSIZE]; + char buf[MAX_DEBUG_BUFSIZE], *pos = buf; + int max = MAX_DEBUG_BUFSIZE; va_start(args, msg); if (dbgf) { - if (bvsnprintf(buf, MAX_DEBUG_BUFSIZE, msg, args) < 0) + struct timespec dbg_time; + clock_gettime(CLOCK_MONOTONIC, &dbg_time); + uint nsec; + uint sec; + + if (dbg_time.tv_nsec > dbg_time_start.tv_nsec) + { + nsec = dbg_time.tv_nsec - dbg_time_start.tv_nsec; + sec = dbg_time.tv_sec - dbg_time_start.tv_sec; + } + else + { + nsec = 1000000000 + dbg_time.tv_nsec - dbg_time_start.tv_nsec; + sec = dbg_time.tv_sec - dbg_time_start.tv_sec - 1; + } + + int n = bsnprintf(pos, max, "%u.%09u: [%04x] ", sec, nsec, THIS_CORO_ID); + pos += n; + max -= n; + + if (bvsnprintf(pos, max, msg, args) < 0) bug("Extremely long debug output, split it."); fputs(buf, dbgf); @@ -422,6 +439,8 @@ done: void log_init_debug(char *f) { + clock_gettime(CLOCK_MONOTONIC, &dbg_time_start); + if (dbgf && dbgf != stderr) fclose(dbgf); if (!f) diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index 07d6c691..8bc07d94 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -28,6 +28,7 @@ #include "lib/resource.h" #include "lib/socket.h" #include "lib/event.h" +#include "lib/locking.h" #include "lib/timer.h" #include "lib/string.h" #include "nest/rt.h" @@ -873,6 +874,7 @@ main(int argc, char **argv) dmalloc_debug(0x2f03d00); #endif + times_update(); parse_args(argc, argv); log_switch(1, NULL, NULL); @@ -927,6 +929,8 @@ main(int argc, char **argv) dup2(0, 2); } + the_bird_lock(); + main_thread_init(); write_pid_file(); diff --git a/sysdep/unix/unix.h b/sysdep/unix/unix.h index ad85d1ea..313c97c3 100644 --- a/sysdep/unix/unix.h +++ b/sysdep/unix/unix.h @@ -106,6 +106,7 @@ extern volatile sig_atomic_t async_shutdown_flag; void io_init(void); void io_loop(void); +void io_loop_reload(void); void io_log_dump(void); int sk_open_unix(struct birdsock *s, char *name); struct rfile *rf_open(struct pool *, const char *name, const char *mode);