Merge commit '94eb0858' into thread-next

filter/filter_test.c

@@ -70,6 +70,7 @@ int
 main(int argc, char *argv[])
 {
   bt_init(argc, argv);
+
   bt_bird_init();
 
   bt_assert_hook = bt_assert_filter;

lib/a-set_test.c

@@ -221,6 +221,7 @@ t_set_ec_delete(void)
   return 1;
 }
 
+
 int
 main(int argc, char *argv[])
 {

lib/birdlib.h

@@ -86,6 +86,7 @@ static inline int u64_cmp(u64 i1, u64 i2)
 /* Macros for gcc attributes */
 
 #define NORET __attribute__((noreturn))
+#define USE_RESULT __atribute__((warn_unused_result))
 #define UNUSED __attribute__((unused))
 #define PACKED __attribute__((packed))
 #define NONNULL(...) __attribute__((nonnull((__VA_ARGS__))))

lib/coro.h

@@ -2,7 +2,7 @@
  *	BIRD Coroutines
  *
  *	(c) 2017 Martin Mares <mj@ucw.cz>
- *	(c) 2020 Maria Matejka <mq@jmq.cz>
+ *	(c) 2020-2021 Maria Matejka <mq@jmq.cz>
  *
  *	Can be freely distributed and used under the terms of the GNU GPL.
  */
@@ -22,5 +22,8 @@ struct coroutine;
  */
 struct coroutine *coro_run(pool *, void (*entry)(void *), void *data);
 
+/* Get self. */
+extern _Thread_local struct coroutine *this_coro;
+
 
 #endif

lib/event.c

@@ -19,8 +19,14 @@
  * events in them and explicitly ask to run them.
  */
 
+#undef LOCAL_DEBUG
+
 #include "nest/bird.h"
 #include "lib/event.h"
+#include "lib/locking.h"
+#include "lib/io-loop.h"
+
+extern _Thread_local struct coroutine *this_coro;
 
 event_list global_event_list;
 event_list global_work_list;
@@ -28,11 +34,16 @@ event_list global_work_list;
 inline void
 ev_postpone(event *e)
 {
+  event_list *el = e->list;
+  if (!el)
+    return;
+
+  ASSERT_DIE(birdloop_inside(el->loop));
+
+  LOCK_DOMAIN(event, el->lock);
   if (ev_active(e))
-    {
+    rem_node(&e->n);
-      rem_node(&e->n);
+  UNLOCK_DOMAIN(event, el->lock);
-      e->n.next = NULL;
-    }
 }
 
 static void
@@ -95,40 +106,25 @@ ev_run(event *e)
  * list @l which can be run by calling ev_run_list().
  */
 inline void
-ev_enqueue(event_list *l, event *e)
+ev_send(event_list *l, event *e)
 {
-  ev_postpone(e);
+  DBG("ev_send(%p, %p)\n", l, e);
-  add_tail(l, &e->n);
+  ASSERT_DIE(e->hook);
-}
+  ASSERT_DIE(!e->list || (e->list == l) || (e->list->loop == l->loop));
 
-/**
+  e->list = l;
- * ev_schedule - schedule an event
- * @e: an event
- *
- * This function schedules an event by enqueueing it to a system-wide
- * event list which is run by the platform dependent code whenever
- * appropriate.
- */
-void
-ev_schedule(event *e)
-{
-  ev_enqueue(&global_event_list, e);
-}
 
-/**
+  LOCK_DOMAIN(event, l->lock);
- * ev_schedule_work - schedule a work-event.
+  if (enlisted(&e->n))
- * @e: an event
+  {
- *
+    UNLOCK_DOMAIN(event, l->lock);
- * This function schedules an event by enqueueing it to a system-wide work-event
+    return;
- * list which is run by the platform dependent code whenever appropriate. This
+  }
- * is designated for work-events instead of regular events. They are executed
+
- * less often in order to not clog I/O loop.
+  add_tail(&l->events, &e->n);
- */
+  UNLOCK_DOMAIN(event, l->lock);
-void
+
-ev_schedule_work(event *e)
+  birdloop_ping(l->loop);
-{
-  if (!ev_active(e))
-    add_tail(&global_work_list, &e->n);
 }
 
 void io_log_event(void *hook, void *data);
@@ -142,36 +138,65 @@ void io_log_event(void *hook, void *data);
 int
 ev_run_list(event_list *l)
 {
-  node *n;
+  const _Bool legacy = LEGACY_EVENT_LIST(l);
-  list tmp_list;
 
+  if (legacy)
+    ASSERT_THE_BIRD_LOCKED;
+
+  node *n;
+
+  list tmp_list;
   init_list(&tmp_list);
-  add_tail_list(&tmp_list, l);
+
-  init_list(l);
+  /* Move the event list contents to a local list to avoid executing repeatedly added events */
+  LOCK_DOMAIN(event, l->lock);
+  add_tail_list(&tmp_list, &l->events);
+  init_list(&l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+
   WALK_LIST_FIRST(n, tmp_list)
     {
       event *e = SKIP_BACK(event, n, n);
 
-      /* This is ugly hack, we want to log just events executed from the main I/O loop */
+      if (legacy)
-      if ((l == &global_event_list) || (l == &global_work_list))
+      {
+	/* The legacy way of event execution */
 	io_log_event(e->hook, e->data);
-
+	ev_postpone(e);
-      ev_run(e);
+	e->hook(e->data);
+      }
+      else
+      {
+	// io_log_event(e->hook, e->data); /* TODO: add support for event logging in other io loops */
+	ASSERT_DIE(e->list == l);
+	LOCK_DOMAIN(event, l->lock);
+	rem_node(&e->n);
+	UNLOCK_DOMAIN(event, l->lock);
+	e->hook(e->data);
+      }
       tmp_flush();
     }
 
-  return !EMPTY_LIST(*l);
+  LOCK_DOMAIN(event, l->lock);
+  int repeat = ! EMPTY_LIST(l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+  return repeat;
 }
 
 int
 ev_run_list_limited(event_list *l, uint limit)
 {
+  ASSERT_DIE(LEGACY_EVENT_LIST(l));
+  ASSERT_THE_BIRD_LOCKED;
+
   node *n;
   list tmp_list;
 
+  LOCK_DOMAIN(event, l->lock);
   init_list(&tmp_list);
-  add_tail_list(&tmp_list, l);
+  add_tail_list(&tmp_list, &l->events);
-  init_list(l);
+  init_list(&l->events);
+  UNLOCK_DOMAIN(event, l->lock);
 
   WALK_LIST_FIRST(n, tmp_list)
     {
@@ -180,22 +205,24 @@ ev_run_list_limited(event_list *l, uint limit)
       if (!limit)
 	break;
 
-      /* This is ugly hack, we want to log just events executed from the main I/O loop */
+      io_log_event(e->hook, e->data);
-      if ((l == &global_event_list) || (l == &global_work_list))
-	io_log_event(e->hook, e->data);
 
       ev_run(e);
       tmp_flush();
       limit--;
     }
 
+  LOCK_DOMAIN(event, l->lock);
   if (!EMPTY_LIST(tmp_list))
   {
     /* Attach new items after the unprocessed old items */
-    add_tail_list(&tmp_list, l);
+    add_tail_list(&tmp_list, &l->events);
-    init_list(l);
+    init_list(&l->events);
-    add_tail_list(l, &tmp_list);
+    add_tail_list(&l->events, &tmp_list);
   }
 
-  return !EMPTY_LIST(*l);
+  int repeat = ! EMPTY_LIST(l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+
+  return repeat;
 }

lib/event.h

@@ -10,33 +10,62 @@
 #define _BIRD_EVENT_H_
 
 #include "lib/resource.h"
+#include "lib/locking.h"
+
+#include <stdatomic.h>
+
+DEFINE_DOMAIN(event);
 
 typedef struct event {
   resource r;
   void (*hook)(void *);
   void *data;
   node n;				/* Internal link */
+  struct event_list *list;		/* List where this event is put in */
 } event;
 
-typedef list event_list;
+typedef struct event_list {
+  list events;
+  pool *pool;
+  struct birdloop *loop;
+  DOMAIN(event) lock;
+} event_list;
 
 extern event_list global_event_list;
 extern event_list global_work_list;
 
 event *ev_new(pool *);
 void ev_run(event *);
-#define ev_init_list(el) init_list(el)
+
-void ev_enqueue(event_list *, event *);
+static inline void ev_init_list(event_list *el, struct birdloop *loop, const char *name)
-void ev_schedule(event *);
+{
-void ev_schedule_work(event *);
+  init_list(&el->events);
+  el->loop = loop;
+  el->lock = DOMAIN_NEW(event, name);
+}
+
+void ev_send(event_list *, event *);
+#define ev_send_loop(l, e) ev_send(birdloop_event_list((l)), (e))
+
+#define ev_schedule(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_event_list, (e)); })
+#define ev_schedule_work(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_work_list, (e)); })
+
 void ev_postpone(event *);
 int ev_run_list(event_list *);
 int ev_run_list_limited(event_list *, uint);
 
+#define LEGACY_EVENT_LIST(l)  (((l) == &global_event_list) || ((l) == &global_work_list))
+
+_Bool birdloop_inside(struct birdloop *loop);
+
 static inline int
 ev_active(event *e)
 {
-  return e->n.next != NULL;
+  if (e->list == NULL)
+    return 0;
+
+  ASSERT_DIE(birdloop_inside(e->list->loop));
+  return enlisted(&e->n);
 }
 
 static inline event*

lib/event_test.c

@@ -54,7 +54,7 @@ t_ev_run_list(void)
   int i;
 
   olock_init();
-  timer_init();
+  birdloop_init();
   rt_init();
   io_init();
   if_init();
@@ -81,7 +81,9 @@ main(int argc, char *argv[])
 {
   bt_init(argc, argv);
 
+  the_bird_lock();
   bt_test_suite(t_ev_run_list, "Schedule and run 3 events in right order.");
+  the_bird_unlock();
 
   return bt_exit_value();
 }

lib/io-loop.h

@@ -0,0 +1,54 @@
+/*
+ *	BIRD -- I/O and event loop
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL.
+ */
+
+#ifndef _BIRD_IO_LOOP_H_
+#define _BIRD_IO_LOOP_H_
+
+#include "nest/bird.h"
+#include "lib/lists.h"
+#include "lib/locking.h"
+#include "lib/resource.h"
+#include "lib/event.h"
+#include "lib/socket.h"
+
+void sk_start(sock *s);
+void sk_stop(sock *s);
+void sk_reloop(sock *s, struct birdloop *loop);
+
+extern struct birdloop main_birdloop;
+
+/* Start a new birdloop owned by given pool and domain */
+struct birdloop *birdloop_new(pool *p, uint order, const char *name);
+
+/* Stop the loop. At the end, the @stopped callback is called unlocked in tail
+ * position to finish cleanup. Run birdloop_free() from that callback to free
+ * the loop itself. */
+void birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data);
+void birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data);
+void birdloop_free(struct birdloop *loop);
+
+/* Get birdloop's event list */
+event_list *birdloop_event_list(struct birdloop *loop);
+
+/* Get birdloop's time heap */
+struct timeloop *birdloop_time_loop(struct birdloop *loop);
+
+/* Enter and exit the birdloop */
+void birdloop_enter(struct birdloop *loop);
+void birdloop_leave(struct birdloop *loop);
+
+_Bool birdloop_inside(struct birdloop *loop);
+
+void birdloop_mask_wakeups(struct birdloop *loop);
+void birdloop_unmask_wakeups(struct birdloop *loop);
+
+void birdloop_link(struct birdloop *loop);
+void birdloop_unlink(struct birdloop *loop);
+
+void birdloop_ping(struct birdloop *loop);
+
+void birdloop_init(void);
+#endif /* _BIRD_IO_LOOP_H_ */

lib/lists.h

@@ -69,6 +69,18 @@ typedef union list {			/* In fact two overlayed nodes */
 
 #define EMPTY_LIST(list) (!(list).head->next)
 
+static inline _Bool
+enlisted(node *n)
+{
+  switch ((!!n->next) + (!!n->prev))
+  {
+    case 0: return 0;
+    case 2: return 1;
+    case 1: bug("Garbled event list node");
+  }
+
+  bug("Maths is broken. And you should see a new heaven and a new earth: for the first heaven and the first earth had been passed away.");
+}
 
 #ifndef _BIRD_LISTS_C_
 #define LIST_INLINE static inline

lib/locking.h

@@ -14,6 +14,9 @@ struct domain_generic;
 /* Here define the global lock order; first to last. */
 struct lock_order {
   struct domain_generic *the_bird;
+  struct domain_generic *proto;
+  struct domain_generic *rtable;
+  struct domain_generic *event;
 };
 
 extern _Thread_local struct lock_order locking_stack;
@@ -21,24 +24,40 @@ extern _Thread_local struct domain_generic **last_locked;
 
 #define DOMAIN(type) struct domain__##type
 #define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; }
+#define DOMAIN_ORDER(type)  OFFSETOF(struct lock_order, type)
 
-#define DOMAIN_NEW(type, name)  (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) }
+#define DOMAIN_NEW(type, name)  (DOMAIN(type)) { .type = domain_new(name, DOMAIN_ORDER(type)) }
 struct domain_generic *domain_new(const char *name, uint order);
 
+#define DOMAIN_FREE(type, d)	domain_free((d).type)
+void domain_free(struct domain_generic *);
+
 #define DOMAIN_NULL(type)   (DOMAIN(type)) {}
 
 #define LOCK_DOMAIN(type, d)	do_lock(((d).type), &(locking_stack.type))
 #define UNLOCK_DOMAIN(type, d)  do_unlock(((d).type), &(locking_stack.type))
 
+#define DOMAIN_IS_LOCKED(type, d) (((d).type) == (locking_stack.type))
+#define DG_IS_LOCKED(d)	((d) == *(DG_LSP(d)))
+
 /* Internal for locking */
 void do_lock(struct domain_generic *dg, struct domain_generic **lsp);
 void do_unlock(struct domain_generic *dg, struct domain_generic **lsp);
 
+uint dg_order(struct domain_generic *dg);
+
+#define DG_LSP(d)	((struct domain_generic **) (((void *) &locking_stack) + dg_order(d)))
+#define DG_LOCK(d)	do_lock(d, DG_LSP(d))
+#define DG_UNLOCK(d)	do_unlock(d, DG_LSP(d))
+
 /* Use with care. To be removed in near future. */
 DEFINE_DOMAIN(the_bird);
 extern DOMAIN(the_bird) the_bird_domain;
 
 #define the_bird_lock()		LOCK_DOMAIN(the_bird, the_bird_domain)
 #define the_bird_unlock()	UNLOCK_DOMAIN(the_bird, the_bird_domain)
+#define the_bird_locked()	DOMAIN_IS_LOCKED(the_bird, the_bird_domain)
+
+#define ASSERT_THE_BIRD_LOCKED	({ if (!the_bird_locked()) bug("The BIRD lock must be locked here: %s:%d", __FILE__, __LINE__); })
 
 #endif

lib/resource.c

@@ -29,12 +29,6 @@
  * is freed upon shutdown of the module.
  */
 
-struct pool {
-  resource r;
-  list inside;
-  const char *name;
-};
-
 static void pool_dump(resource *);
 static void pool_free(resource *);
 static resource *pool_lookup(resource *, unsigned long);

lib/resource.h

@@ -40,7 +40,12 @@ struct resclass {
 
 /* Generic resource manipulation */
 
-typedef struct pool pool;
+typedef struct pool {
+  resource r;
+  list inside;
+  const char *name;
+} pool;
+
 
 void resource_init(void);
 pool *rp_new(pool *, const char *);	/* Create new pool */

lib/socket.h

@@ -12,6 +12,7 @@
 #include <errno.h>
 
 #include "lib/resource.h"
+#include "lib/event.h"
 #ifdef HAVE_LIBSSH
 #define LIBSSH_LEGACY_0_4
 #include <libssh/libssh.h>
@@ -79,6 +80,7 @@ typedef struct birdsock {
   const char *password;			/* Password for MD5 authentication */
   const char *err;			/* Error message */
   struct ssh_sock *ssh;			/* Used in SK_SSH */
+  struct event reloop;			/* Reloop event */
 } sock;
 
 sock *sock_new(pool *);			/* Allocate new socket */
@@ -129,6 +131,7 @@ extern int sk_priority_control;		/* Suggested priority for control traffic, shou
 #define SKF_TRUNCATED	0x200	/* Received packet was truncated, set by IO layer */
 #define SKF_HDRINCL	0x400	/* Used internally */
 #define SKF_PKTINFO	0x800	/* Used internally */
+#define SKF_PASSIVE_THREAD  0x1000  /* Child sockets used in thread, do not add to main loop */
 
 /*
  *	Socket types		     SA SP DA DP IF  TTL SendTo	(?=may, -=must not, *=must)

lib/timer.c

@@ -37,15 +37,8 @@
 #include "lib/resource.h"
 #include "lib/timer.h"
 
-
-struct timeloop main_timeloop;
-
-
 #include <pthread.h>
 
-/* Data accessed and modified from proto/bfd/io.c */
-_Thread_local struct timeloop *local_timeloop;
-
 _Atomic btime last_time;
 _Atomic btime real_time;
 
@@ -76,7 +69,7 @@ tm_dump(resource *r)
   if (t->recurrent)
     debug("recur %d, ", t->recurrent);
   if (t->expires)
-    debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS);
+    debug("in loop %p expires in %d ms)\n", t->loop, (t->expires - current_time()) TO_MS);
   else
     debug("inactive)\n");
 }
@@ -99,8 +92,8 @@ tm_new(pool *p)
   return t;
 }
 
-void
+static void
-tm_set(timer *t, btime when)
+tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop)
 {
   uint tc = timers_count(local_timeloop);
 
@@ -122,17 +115,17 @@ tm_set(timer *t, btime when)
     HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
   }
 
-#ifdef CONFIG_BFD
+  t->loop = local_timeloop;
-  /* Hack to notify BFD loops */
+
-  if ((local_timeloop != &main_timeloop) && (t->index == 1))
+  if ((t->index == 1) && (local_timeloop->coro != this_coro))
-    wakeup_kick_current();
+    birdloop_ping(local_timeloop->loop);
-#endif
 }
 
 void
-tm_start(timer *t, btime after)
+tm_set_in(timer *t, btime when, struct birdloop *loop)
 {
-  tm_set(t, current_time() + MAX(after, 0));
+  ASSERT_DIE(birdloop_inside(loop));
+  tm_set_in_tl(t, when, birdloop_time_loop(loop));
 }
 
 void
@@ -141,18 +134,23 @@ tm_stop(timer *t)
   if (!t->expires)
     return;
 
-  uint tc = timers_count(local_timeloop);
+  TLOCK_TIMER_ASSERT(t->loop);
 
-  HEAP_DELETE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
+  uint tc = timers_count(t->loop);
-  BUFFER_POP(local_timeloop->timers);
+
+  HEAP_DELETE(t->loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
+  BUFFER_POP(t->loop->timers);
 
   t->index = -1;
   t->expires = 0;
+  t->loop = NULL;
 }
 
 void
 timers_init(struct timeloop *loop, pool *p)
 {
+  TLOCK_TIMER_ASSERT(loop);
+
   BUFFER_INIT(loop->timers, p, 4);
   BUFFER_PUSH(loop->timers) = NULL;
 }
@@ -160,8 +158,10 @@ timers_init(struct timeloop *loop, pool *p)
 void io_log_event(void *hook, void *data);
 
 void
-timers_fire(struct timeloop *loop)
+timers_fire(struct timeloop *loop, int io_log)
 {
+  TLOCK_TIMER_ASSERT(loop);
+
   btime base_time;
   timer *t;
 
@@ -183,13 +183,13 @@ timers_fire(struct timeloop *loop)
       if (t->randomize)
 	when += random() % (t->randomize + 1);
 
-      tm_set(t, when);
+      tm_set_in_tl(t, when, loop);
     }
     else
       tm_stop(t);
 
     /* This is ugly hack, we want to log just timers executed from the main I/O loop */
-    if (loop == &main_timeloop)
+    if (io_log)
       io_log_event(t->hook, t->data);
 
     t->hook(t);
@@ -197,13 +197,6 @@ timers_fire(struct timeloop *loop)
   }
 }
 
-void
-timer_init(void)
-{
-  timers_init(&main_timeloop, &root_pool);
-  local_timeloop = &main_timeloop;
-}
-
 
 /**
  * tm_parse_time - parse a date and time

lib/timer.h

@@ -12,6 +12,8 @@
 
 #include "nest/bird.h"
 #include "lib/buffer.h"
+#include "lib/io-loop.h"
+#include "lib/locking.h"
 #include "lib/resource.h"
 
 #include <stdatomic.h>
@@ -29,22 +31,27 @@ typedef struct timer
   uint randomize;			/* Amount of randomization */
   uint recurrent;			/* Timer recurrence */
 
+  struct timeloop *loop;		/* Loop where the timer is active */
+
   int index;
 } timer;
 
 struct timeloop
 {
   BUFFER_(timer *) timers;
+  struct domain_generic *domain;
+  struct birdloop *loop;
+  struct coroutine *coro;
 };
 
+#define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain))
+#define TLOCK_LOCAL_ASSERT(loop) ASSERT_DIE(!(loop)->domain || DG_IS_LOCKED((loop)->domain))
+
 static inline uint timers_count(struct timeloop *loop)
-{ return loop->timers.used - 1; }
+{ TLOCK_TIMER_ASSERT(loop); return loop->timers.used - 1; }
 
 static inline timer *timers_first(struct timeloop *loop)
-{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; }
+{ TLOCK_TIMER_ASSERT(loop); return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; }
-
-extern struct timeloop main_timeloop;
-extern _Thread_local struct timeloop *local_timeloop;
 
 #define current_time()		atomic_load_explicit(&last_time, memory_order_acquire)
 #define current_real_time()	atomic_load_explicit(&real_time, memory_order_acquire)
@@ -54,10 +61,13 @@ extern _Thread_local struct timeloop *local_timeloop;
 extern btime boot_time;
 
 timer *tm_new(pool *p);
-void tm_set(timer *t, btime when);
+#define tm_set(t, when) tm_set_in((t), (when), &main_birdloop)
-void tm_start(timer *t, btime after);
+#define tm_start(t, after) tm_start_in((t), (after), &main_birdloop)
 void tm_stop(timer *t);
 
+void tm_set_in(timer *t, btime when, struct birdloop *loop);
+#define tm_start_in(t, after, loop) tm_set_in((t), (current_time() + MAX_((after), 0)), loop)
+
 static inline int
 tm_active(timer *t)
 {
@@ -101,9 +111,7 @@ void times_update(void);
 
 /* For I/O loop */
 void timers_init(struct timeloop *loop, pool *p);
-void timers_fire(struct timeloop *loop);
+void timers_fire(struct timeloop *loop, int io_log);
-
-void timer_init(void);
 
 
 struct timeformat {

nest/config.Y

@@ -472,6 +472,7 @@ proto: dev_proto '}' ;
 dev_proto_start: proto_start DIRECT {
      this_proto = proto_config_new(&proto_device, $1);
      init_list(&DIRECT_CFG->iface_list);
+     this_proto->late_if_feed = 1;
    }
  ;
 

nest/proto.c

@@ -15,6 +15,7 @@
 #include "lib/event.h"
 #include "lib/timer.h"
 #include "lib/string.h"
+#include "lib/coro.h"
 #include "conf/conf.h"
 #include "nest/rt.h"
 #include "nest/iface.h"
@@ -57,7 +58,28 @@ static void channel_feed_end(struct channel *c);
 static void channel_export_stopped(struct rt_export_request *req);
 
 static inline int proto_is_done(struct proto *p)
-{ return (p->proto_state == PS_DOWN) && (p->active_channels == 0); }
+{ return (p->proto_state == PS_DOWN) && proto_is_inactive(p); }
+
+static inline event_list *proto_event_list(struct proto *p)
+{ return p->loop == &main_birdloop ? &global_event_list : birdloop_event_list(p->loop); }
+
+static inline event_list *proto_work_list(struct proto *p)
+{ return p->loop == &main_birdloop ? &global_work_list : birdloop_event_list(p->loop); }
+
+static inline void proto_send_event(struct proto *p)
+{ ev_send(proto_event_list(p), p->event); }
+
+#define PROTO_ENTER_FROM_MAIN(p)    ({ \
+    ASSERT_DIE(birdloop_inside(&main_birdloop)); \
+    struct birdloop *_loop = (p)->loop; \
+    if (_loop != &main_birdloop) birdloop_enter(_loop); \
+    _loop; \
+    })
+
+#define PROTO_LEAVE_FROM_MAIN(loop) ({ if (loop != &main_birdloop) birdloop_leave(loop); })
+
+#define PROTO_LOCKED_FROM_MAIN(p)	for (struct birdloop *_proto_loop = PROTO_ENTER_FROM_MAIN(p); _proto_loop; PROTO_LEAVE_FROM_MAIN(_proto_loop), (_proto_loop = NULL))
+
 
 static inline int channel_is_active(struct channel *c)
 { return (c->channel_state != CS_DOWN); }
@@ -467,6 +489,7 @@ channel_start_export(struct channel *c)
 
   c->out_req = (struct rt_export_request) {
     .name = rn,
+    .list = proto_work_list(c->proto),
     .addr = c->out_subprefix,
     .addr_mode = c->out_subprefix ? TE_ADDR_IN : TE_ADDR_NONE,
     .trace_routes = c->debug | c->proto->debug,
@@ -513,7 +536,7 @@ channel_check_stopped(struct channel *c)
 	return;
 
       channel_set_state(c, CS_DOWN);
-      ev_schedule(c->proto->event);
+      proto_send_event(c->proto);
 
       break;
     case CS_PAUSE:
@@ -633,6 +656,7 @@ channel_setup_in_table(struct channel *c)
 {
   c->reload_req = (struct rt_export_request) {
     .name = mb_sprintf(c->proto->pool, "%s.%s.import", c->proto->name, c->name),
+    .list = proto_work_list(c->proto),
     .trace_routes = c->debug | c->proto->debug,
     .export_bulk = channel_reload_export_bulk,
     .dump_req = channel_reload_dump_req,
@@ -718,7 +742,7 @@ channel_do_down(struct channel *c)
 
   /* Schedule protocol shutddown */
   if (proto_is_done(c->proto))
-    ev_schedule(c->proto->event);
+    proto_send_event(c->proto);
 }
 
 void
@@ -1029,18 +1053,36 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con
   return 1;
 }
 
+static void
+proto_cleanup(struct proto *p)
+{
+  rfree(p->pool);
+  p->pool = NULL;
+
+  p->active = 0;
+  proto_log_state_change(p);
+  proto_rethink_goal(p);
+}
+
+static void
+proto_loop_stopped(void *ptr)
+{
+  struct proto *p = ptr;
+
+  birdloop_enter(&main_birdloop);
+
+  p->loop = &main_birdloop;
+  p->event->list = NULL;
+  proto_cleanup(p);
+
+  birdloop_leave(&main_birdloop);
+}
 
 static void
 proto_event(void *ptr)
 {
   struct proto *p = ptr;
 
-  if (p->do_start)
-  {
-    if_feed_baby(p);
-    p->do_start = 0;
-  }
-
   if (p->do_stop)
   {
     if (p->proto == &proto_unix_iface)
@@ -1049,14 +1091,10 @@ proto_event(void *ptr)
   }
 
   if (proto_is_done(p))
-  {
+    if (p->loop != &main_birdloop)
-    rfree(p->pool);
+      birdloop_stop_self(p->loop, proto_loop_stopped, p);
-    p->pool = NULL;
+    else
-
+      proto_cleanup(p);
-    p->active = 0;
-    proto_log_state_change(p);
-    proto_rethink_goal(p);
-  }
 }
 
 
@@ -1097,6 +1135,7 @@ proto_init(struct proto_config *c, node *n)
   struct protocol *pr = c->protocol;
   struct proto *p = pr->init(c);
 
+  p->loop = &main_birdloop;
   p->proto_state = PS_DOWN;
   p->last_state_change = current_time();
   p->vrf = c->vrf;
@@ -1113,11 +1152,21 @@ proto_init(struct proto_config *c, node *n)
 static void
 proto_start(struct proto *p)
 {
-  /* Here we cannot use p->cf->name since it won't survive reconfiguration */
+  DBG("Kicking %s up\n", p->name);
-  p->pool = rp_new(proto_pool, p->proto->name);
+  PD(p, "Starting");
+
+  p->pool = rp_newf(proto_pool, "Protocol %s", p->cf->name);
 
   if (graceful_restart_state == GRS_INIT)
     p->gr_recovery = 1;
+
+  if (p->cf->loop_order != DOMAIN_ORDER(the_bird))
+    p->loop = birdloop_new(p->pool, p->cf->loop_order, p->pool->name);
+
+  p->event->list = proto_event_list(p);
+
+  PROTO_LOCKED_FROM_MAIN(p)
+    proto_notify_state(p, (p->proto->start ? p->proto->start(p) : PS_UP));
 }
 
 
@@ -1153,6 +1202,7 @@ proto_config_new(struct protocol *pr, int class)
   cf->class = class;
   cf->debug = new_config->proto_default_debug;
   cf->mrtdump = new_config->proto_default_mrtdump;
+  cf->loop_order = DOMAIN_ORDER(the_bird);
 
   init_list(&cf->channels);
 
@@ -1441,12 +1491,21 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty
     proto_rethink_goal(p);
 }
 
+static void
+proto_shutdown(struct proto *p)
+{
+  if (p->proto_state == PS_START || p->proto_state == PS_UP)
+  {
+    /* Going down */
+    DBG("Kicking %s down\n", p->name);
+    PD(p, "Shutting down");
+    proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN));
+  }
+}
+
 static void
 proto_rethink_goal(struct proto *p)
 {
-  struct protocol *q;
-  byte goal;
-
   if (p->reconfiguring && !p->active)
   {
     struct proto_config *nc = p->cf_new;
@@ -1466,32 +1525,12 @@ proto_rethink_goal(struct proto *p)
 
   /* Determine what state we want to reach */
   if (p->disabled || p->reconfiguring)
-    goal = PS_DOWN;
-  else
-    goal = PS_UP;
-
-  q = p->proto;
-  if (goal == PS_UP)
   {
-    if (!p->active)
+    PROTO_LOCKED_FROM_MAIN(p)
-    {
+      proto_shutdown(p);
-      /* Going up */
-      DBG("Kicking %s up\n", p->name);
-      PD(p, "Starting");
-      proto_start(p);
-      proto_notify_state(p, (q->start ? q->start(p) : PS_UP));
-    }
-  }
-  else
-  {
-    if (p->proto_state == PS_START || p->proto_state == PS_UP)
-    {
-      /* Going down */
-      DBG("Kicking %s down\n", p->name);
-      PD(p, "Shutting down");
-      proto_notify_state(p, (q->shutdown ? q->shutdown(p) : PS_DOWN));
-    }
   }
+  else if (!p->active)
+    proto_start(p);
 }
 
 struct proto *
@@ -1696,7 +1735,7 @@ protos_dump_all(void)
 #define DPF(x)	(p->x ? " " #x : "")
     debug("  protocol %s (%p) state %s with %d active channels flags: %s%s%s%s%s\n",
 	p->name, p, p_states[p->proto_state], p->active_channels,
-	DPF(disabled), DPF(active), DPF(do_start), DPF(do_stop), DPF(reconfiguring));
+	DPF(disabled), DPF(active), DPF(do_stop), DPF(reconfiguring));
 #undef DPF
 
     struct channel *c;
@@ -1932,8 +1971,8 @@ static inline void
 proto_do_start(struct proto *p)
 {
   p->active = 1;
-  p->do_start = 1;
+  if (!p->cf->late_if_feed)
-  ev_schedule(p->event);
+    if_feed_baby(p);
 }
 
 static void
@@ -1946,6 +1985,9 @@ proto_do_up(struct proto *p)
   }
 
   proto_start_channels(p);
+
+  if (p->cf->late_if_feed)
+    if_feed_baby(p);
 }
 
 static inline void
@@ -1960,9 +2002,6 @@ proto_do_stop(struct proto *p)
   p->down_sched = 0;
   p->gr_recovery = 0;
 
-  p->do_stop = 1;
-  ev_schedule(p->event);
-
   if (p->main_source)
   {
     rt_unlock_source(p->main_source);
@@ -1970,6 +2009,9 @@ proto_do_stop(struct proto *p)
   }
 
   proto_stop_channels(p);
+
+  p->do_stop = 1;
+  proto_send_event(p);
 }
 
 static void
@@ -1980,7 +2022,7 @@ proto_do_down(struct proto *p)
 
   /* Shutdown is finished in the protocol event */
   if (proto_is_done(p))
-    ev_schedule(p->event);
+    proto_send_event(p);
 }
 
 
@@ -2219,7 +2261,7 @@ proto_cmd_disable(struct proto *p, uintptr_t arg, int cnt UNUSED)
   p->disabled = 1;
   p->down_code = PDC_CMD_DISABLE;
   proto_set_message(p, (char *) arg, -1);
-  proto_rethink_goal(p);
+  proto_shutdown(p);
   cli_msg(-9, "%s: disabled", p->name);
 }
 
@@ -2252,9 +2294,9 @@ proto_cmd_restart(struct proto *p, uintptr_t arg, int cnt UNUSED)
   p->disabled = 1;
   p->down_code = PDC_CMD_RESTART;
   proto_set_message(p, (char *) arg, -1);
-  proto_rethink_goal(p);
+  proto_shutdown(p);
   p->disabled = 0;
-  proto_rethink_goal(p);
+  /* After the protocol shuts down, proto_rethink_goal() is run from proto_event. */
   cli_msg(-12, "%s: restarted", p->name);
 }
 
@@ -2329,7 +2371,9 @@ proto_apply_cmd_symbol(const struct symbol *s, void (* cmd)(struct proto *, uint
 
   if (s->proto->proto)
   {
-    cmd(s->proto->proto, arg, 0);
+    struct proto *p = s->proto->proto;
+    PROTO_LOCKED_FROM_MAIN(p)
+      cmd(p, arg, 0);
     cli_msg(0, "");
   }
   else
@@ -2344,7 +2388,8 @@ proto_apply_cmd_patt(const char *patt, void (* cmd)(struct proto *, uintptr_t, i
 
   WALK_LIST(p, proto_list)
     if (!patt || patmatch(patt, p->name))
-      cmd(p, arg, cnt++);
+      PROTO_LOCKED_FROM_MAIN(p)
+	cmd(p, arg, cnt++);
 
   if (!cnt)
     cli_msg(8003, "No protocols match");

nest/protocol.h

@@ -102,8 +102,10 @@ struct proto_config {
   u8 net_type;				/* Protocol network type (NET_*), 0 for undefined */
   u8 disabled;				/* Protocol enabled/disabled by default */
   u8 vrf_set;				/* Related VRF instance (below) is defined */
+  u8 late_if_feed;			/* Delay interface feed after channels are up */
   u32 debug, mrtdump;			/* Debugging bitfields, both use D_* constants */
   u32 router_id;			/* Protocol specific router ID */
+  uint loop_order;			/* Launch a birdloop on this locking level; use DOMAIN_ORDER(the_bird) for mainloop */
 
   list channels;			/* List of channel configs (struct channel_config) */
   struct iface *vrf;			/* Related VRF instance, NULL if global */
@@ -121,6 +123,7 @@ struct proto {
   struct proto_config *cf_new;		/* Configuration we want to switch to after shutdown (NULL=delete) */
   pool *pool;				/* Pool containing local objects */
   event *event;				/* Protocol event */
+  struct birdloop *loop;		/* BIRDloop running this protocol */
 
   list channels;			/* List of channels to rtables (struct channel) */
   struct channel *main_channel;		/* Primary channel */
@@ -131,12 +134,12 @@ struct proto {
   u32 debug;				/* Debugging flags */
   u32 mrtdump;				/* MRTDump flags */
   uint active_channels;			/* Number of active channels */
+  uint active_coroutines;		/* Number of active coroutines */
   byte net_type;			/* Protocol network type (NET_*), 0 for undefined */
   byte disabled;			/* Manually disabled */
   byte vrf_set;				/* Related VRF instance (above) is defined */
   byte proto_state;			/* Protocol state machine (PS_*, see below) */
   byte active;				/* From PS_START to cleanup after PS_STOP */
-  byte do_start;			/* Start actions are scheduled */
   byte do_stop;				/* Stop actions are scheduled */
   byte reconfiguring;			/* We're shutting down due to reconfiguration */
   byte gr_recovery;			/* Protocol should participate in graceful restart recovery */
@@ -338,6 +341,8 @@ void proto_notify_state(struct proto *p, unsigned state);
  *	as a result of received ROUTE-REFRESH request).
  */
 
+static inline int proto_is_inactive(struct proto *p)
+{ return (p->active_channels == 0) && (p->active_coroutines == 0); }
 
 
 /*

nest/rt-show.c

@@ -285,6 +285,7 @@ rt_show_cont(struct rt_show_data *d)
   d->req = (struct rt_export_request) {
     .addr = d->addr,
     .name = "CLI Show Route",
+    .list = &global_work_list,
     .export_bulk = rt_show_net_export_bulk,
     .dump_req = rt_show_dump_req,
     .log_state_change = rt_show_log_state_change,

nest/rt-table.c

@@ -1415,6 +1415,12 @@ rt_next_export(struct rt_export_hook *hook, struct rt_exporter *tab)
     return tab->first;
 }
 
+static inline void
+rt_send_export_event(struct rt_export_hook *hook)
+{
+  ev_send(hook->req->list, hook->event);
+}
+
 static void
 rt_announce_exports(timer *tm)
 {
@@ -1426,7 +1432,7 @@ rt_announce_exports(timer *tm)
     if (atomic_load_explicit(&c->export_state, memory_order_acquire) != TES_READY)
       continue;
 
-    ev_schedule_work(c->event);
+    rt_send_export_event(c);
   }
 }
 
@@ -1479,7 +1485,7 @@ rt_export_hook(void *_data)
     rte_update_unlock();
   }
 
-  ev_schedule_work(c->event);
+  rt_send_export_event(c);
 }
 
 
@@ -2097,7 +2103,7 @@ rt_request_export(struct rt_exporter *re, struct rt_export_request *req)
 
   /* Regular export */
   rt_set_export_state(hook, TES_FEEDING);
-  ev_schedule_work(hook->event);
+  rt_send_export_event(hook);
 }
 
 static void
@@ -2146,7 +2152,7 @@ rt_stop_export(struct rt_export_request *req, void (*stopped)(struct rt_export_r
   rt_set_export_state(hook, TES_STOP);
 
   /* Run the stopped event */
-  ev_schedule(hook->event);
+  rt_send_export_event(hook);
 }
 
 /**
@@ -3727,7 +3733,7 @@ rt_feed_done(struct rt_export_hook *c)
 
   rt_set_export_state(c, TES_READY);
 
-  ev_schedule_work(c->event);
+  rt_send_export_event(c);
 }
 
 /**
@@ -3756,7 +3762,7 @@ rt_feed_by_fib(void *data)
       if (max_feed <= 0)
 	{
 	  FIB_ITERATE_PUT(fit);
-	  ev_schedule_work(c->event);
+	  rt_send_export_event(c);
 	  return;
 	}
 

nest/rt.h

@@ -17,6 +17,7 @@
 #include "lib/type.h"
 #include "lib/fib.h"
 #include "lib/route.h"
+#include "lib/event.h"
 
 #include <stdatomic.h>
 
@@ -238,6 +239,8 @@ struct rt_export_request {
   u8 trace_routes;
   u8 addr_mode;				/* Network prefilter mode (TE_ADDR_*) */
 
+  event_list *list;			/* Where to schedule export events */
+
   /* There are two methods of export. You can either request feeding every single change
    * or feeding the whole route feed. In case of regular export, &export_one is preferred.
    * Anyway, when feeding, &export_bulk is preferred, falling back to &export_one.

proto/bfd/Makefile

@@ -1,4 +1,4 @@
-src := bfd.c io.c packets.c
+src := bfd.c packets.c
 obj := $(src-o-files)
 $(all-daemon)
 $(cf-local)

proto/bfd/bfd.c

@@ -113,8 +113,16 @@
 #define HASH_IP_EQ(a1,n1,a2,n2)	ipa_equal(a1, a2) && n1 == n2
 #define HASH_IP_FN(a,n)		ipa_hash(a) ^ u32_hash(n)
 
-static list STATIC_LIST_INIT(bfd_proto_list);
+DEFINE_DOMAIN(rtable);
-static list STATIC_LIST_INIT(bfd_wait_list);
+#define BFD_LOCK	LOCK_DOMAIN(rtable, bfd_global.lock)
+#define BFD_UNLOCK	UNLOCK_DOMAIN(rtable, bfd_global.lock)
+
+static struct {
+  DOMAIN(rtable) lock;
+  list wait_list;
+  list pickup_list;
+  list proto_list;
+} bfd_global;
 
 const char *bfd_state_names[] = { "AdminDown", "Down", "Init", "Up" };
 
@@ -188,7 +196,7 @@ bfd_session_update_tx_interval(struct bfd_session *s)
     return;
 
   /* Set timer relative to last tx_timer event */
-  tm_set(s->tx_timer, s->last_tx + tx_int_l);
+  tm_set_in(s->tx_timer, s->last_tx + tx_int_l, s->ifa->bfd->p.loop);
 }
 
 static void
@@ -202,7 +210,7 @@ bfd_session_update_detection_time(struct bfd_session *s, int kick)
   if (!s->last_rx)
     return;
 
-  tm_set(s->hold_timer, s->last_rx + timeout);
+  tm_set_in(s->hold_timer, s->last_rx + timeout, s->ifa->bfd->p.loop);
 }
 
 static void
@@ -226,7 +234,7 @@ bfd_session_control_tx_timer(struct bfd_session *s, int reset)
   if (reset || !tm_active(s->tx_timer))
   {
     s->last_tx = 0;
-    tm_start(s->tx_timer, 0);
+    tm_start_in(s->tx_timer, 0, s->ifa->bfd->p.loop);
   }
 
   return;
@@ -419,7 +427,7 @@ bfd_get_free_id(struct bfd_proto *p)
 static struct bfd_session *
 bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *iface, struct bfd_options *opts)
 {
-  birdloop_enter(p->loop);
+  ASSERT_DIE(birdloop_inside(p->p.loop));
 
   struct bfd_iface *ifa = bfd_get_iface(p, local, iface);
 
@@ -454,8 +462,6 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *
 
   TRACE(D_EVENTS, "Session to %I added", s->addr);
 
-  birdloop_leave(p->loop);
-
   return s;
 }
 
@@ -463,38 +469,34 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *
 static void
 bfd_open_session(struct bfd_proto *p, struct bfd_session *s, ip_addr local, struct iface *ifa)
 {
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
 
   s->opened = 1;
 
   bfd_session_control_tx_timer(s);
 
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 }
 
 static void
 bfd_close_session(struct bfd_proto *p, struct bfd_session *s)
 {
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
 
   s->opened = 0;
 
   bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_PATH_DOWN);
   bfd_session_control_tx_timer(s);
 
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 }
 */
 
 static void
-bfd_remove_session(struct bfd_proto *p, struct bfd_session *s)
+bfd_remove_session_locked(struct bfd_proto *p, struct bfd_session *s)
 {
-  ip_addr ip = s->addr;
-
   /* Caller should ensure that request list is empty */
 
-  birdloop_enter(p->loop);
-
   /* Remove session from notify list if scheduled for notification */
   /* No need for bfd_lock_sessions(), we are already protected by birdloop_enter() */
   if (NODE_VALID(&s->n))
@@ -508,11 +510,17 @@ bfd_remove_session(struct bfd_proto *p, struct bfd_session *s)
   HASH_REMOVE(p->session_hash_id, HASH_ID, s);
   HASH_REMOVE(p->session_hash_ip, HASH_IP, s);
 
+  TRACE(D_EVENTS, "Session to %I removed", s->addr);
+
   sl_free(s);
+}
 
-  TRACE(D_EVENTS, "Session to %I removed", ip);
+static void
-
+bfd_remove_session(struct bfd_proto *p, struct bfd_session *s)
-  birdloop_leave(p->loop);
+{
+  birdloop_enter(p->p.loop);
+  bfd_remove_session_locked(p, s);
+  birdloop_leave(p->p.loop);
 }
 
 static void
@@ -521,7 +529,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s)
   if (EMPTY_LIST(s->request_list))
     return;
 
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
 
   struct bfd_request *req = SKIP_BACK(struct bfd_request, n, HEAD(s->request_list));
   s->cf = bfd_merge_options(s->ifa->cf, &req->opts);
@@ -534,7 +542,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s)
 
   bfd_session_control_tx_timer(s, 0);
 
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 
   TRACE(D_EVENTS, "Session to %I reconfigured", s->addr);
 }
@@ -627,9 +635,9 @@ bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_con
     (new->passive != old->passive);
 
   /* This should be probably changed to not access ifa->cf from the BFD thread */
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
   ifa->cf = new;
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 }
 
 
@@ -690,41 +698,68 @@ bfd_add_request(struct bfd_proto *p, struct bfd_request *req)
 }
 
 static void
-bfd_submit_request(struct bfd_request *req)
+bfd_pickup_requests(void *_data UNUSED)
 {
   node *n;
+  WALK_LIST(n, bfd_global.proto_list)
+  {
+    struct bfd_proto *p = SKIP_BACK(struct bfd_proto, bfd_node, n);
+    birdloop_enter(p->p.loop);
+    BFD_LOCK;
 
-  WALK_LIST(n, bfd_proto_list)
+    node *rn, *rnxt;
-    if (bfd_add_request(SKIP_BACK(struct bfd_proto, bfd_node, n), req))
+    WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list)
-      return;
+      bfd_add_request(p, SKIP_BACK(struct bfd_request, n, rn));
 
-  rem_node(&req->n);
+    BFD_UNLOCK;
-  add_tail(&bfd_wait_list, &req->n);
+    birdloop_leave(p->p.loop);
-  req->session = NULL;
+  }
-  bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0);
+
+  BFD_LOCK;
+  node *rn, *rnxt;
+  WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list)
+  {
+    rem_node(rn);
+    add_tail(&bfd_global.wait_list, rn);
+    bfd_request_notify(SKIP_BACK(struct bfd_request, n, rn), BFD_STATE_ADMIN_DOWN, 0);
+  }
+  BFD_UNLOCK;
 }
 
+static event bfd_pickup_event = { .hook = bfd_pickup_requests };
+
 static void
 bfd_take_requests(struct bfd_proto *p)
 {
   node *n, *nn;
-
+  BFD_LOCK;
-  WALK_LIST_DELSAFE(n, nn, bfd_wait_list)
+  WALK_LIST_DELSAFE(n, nn, bfd_global.wait_list)
     bfd_add_request(p, SKIP_BACK(struct bfd_request, n, n));
+  BFD_UNLOCK;
 }
 
 static void
 bfd_drop_requests(struct bfd_proto *p)
 {
   node *n;
-
+  BFD_LOCK;
-  HASH_WALK(p->session_hash_id, next_id, s)
+  HASH_WALK_DELSAFE(p->session_hash_id, next_id, s)
   {
-    /* We assume that p is not in bfd_proto_list */
     WALK_LIST_FIRST(n, s->request_list)
-      bfd_submit_request(SKIP_BACK(struct bfd_request, n, n));
+    {
+      struct bfd_request *req = SKIP_BACK(struct bfd_request, n, n);
+      rem_node(&req->n);
+      add_tail(&bfd_global.pickup_list, &req->n);
+      req->session = NULL;
+      bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0);
+    }
+
+    ev_send(&global_event_list, &bfd_pickup_event);
+
+    bfd_remove_session_locked(p, s);
   }
   HASH_WALK_END;
+  BFD_UNLOCK;
 }
 
 static struct resclass bfd_request_class;
@@ -737,9 +772,6 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local,
 {
   struct bfd_request *req = ralloc(p, &bfd_request_class);
 
-  /* Hack: self-link req->n, we will call rem_node() on it */
-  req->n.prev = req->n.next = &req->n;
-
   req->addr = addr;
   req->local = local;
   req->iface = iface;
@@ -748,11 +780,16 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local,
   if (opts)
     req->opts = *opts;
 
-  bfd_submit_request(req);
-
   req->hook = hook;
   req->data = data;
 
+  req->session = NULL;
+
+  BFD_LOCK;
+  add_tail(&bfd_global.pickup_list, &req->n);
+  ev_send(&global_event_list, &bfd_pickup_event);
+  BFD_UNLOCK;
+
   return req;
 }
 
@@ -1023,10 +1060,10 @@ bfd_start(struct proto *P)
   struct bfd_proto *p = (struct bfd_proto *) P;
   struct bfd_config *cf = (struct bfd_config *) (P->cf);
 
-  p->loop = birdloop_new();
-  p->tpool = rp_new(NULL, "BFD thread root");
   pthread_spin_init(&p->lock, PTHREAD_PROCESS_PRIVATE);
 
+  p->tpool = rp_new(P->pool, "BFD loop pool");
+
   p->session_slab = sl_new(P->pool, sizeof(struct bfd_session));
   HASH_INIT(p->session_hash_id, P->pool, 8);
   HASH_INIT(p->session_hash_ip, P->pool, 8);
@@ -1036,9 +1073,7 @@ bfd_start(struct proto *P)
   init_list(&p->notify_list);
   bfd_notify_init(p);
 
-  add_tail(&bfd_proto_list, &p->bfd_node);
+  add_tail(&bfd_global.proto_list, &p->bfd_node);
-
-  birdloop_enter(p->loop);
 
   if (!cf->strict_bind)
   {
@@ -1055,42 +1090,33 @@ bfd_start(struct proto *P)
       p->rx6_m = bfd_open_rx_sk(p, 1, SK_IPV6);
   }
 
-  birdloop_leave(p->loop);
-
   bfd_take_requests(p);
 
   struct bfd_neighbor *n;
   WALK_LIST(n, cf->neigh_list)
     bfd_start_neighbor(p, n);
 
-  birdloop_start(p->loop);
-
   return PS_UP;
 }
 
-
 static int
 bfd_shutdown(struct proto *P)
 {
   struct bfd_proto *p = (struct bfd_proto *) P;
-  struct bfd_config *cf = (struct bfd_config *) (P->cf);
+  struct bfd_config *cf = (struct bfd_config *) (p->p.cf);
 
   rem_node(&p->bfd_node);
 
-  birdloop_stop(p->loop);
+  struct bfd_neighbor *bn;
-
+  WALK_LIST(bn, cf->neigh_list)
-  struct bfd_neighbor *n;
+    bfd_stop_neighbor(p, bn);
-  WALK_LIST(n, cf->neigh_list)
-    bfd_stop_neighbor(p, n);
 
   bfd_drop_requests(p);
 
-  /* FIXME: This is hack */
+  if (p->rx4_1) sk_stop(p->rx4_1);
-  birdloop_enter(p->loop);
+  if (p->rx4_m) sk_stop(p->rx4_m);
-  rfree(p->tpool);
+  if (p->rx6_1) sk_stop(p->rx6_1);
-  birdloop_leave(p->loop);
+  if (p->rx6_m) sk_stop(p->rx6_m);
-
-  birdloop_free(p->loop);
 
   return PS_DOWN;
 }
@@ -1111,7 +1137,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c)
       (new->strict_bind != old->strict_bind))
     return 0;
 
-  birdloop_mask_wakeups(p->loop);
+  birdloop_mask_wakeups(p->p.loop);
 
   WALK_LIST(ifa, p->iface_list)
     bfd_reconfigure_iface(p, ifa, new);
@@ -1125,7 +1151,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c)
 
   bfd_reconfigure_neighbors(p, new);
 
-  birdloop_unmask_wakeups(p->loop);
+  birdloop_unmask_wakeups(p->p.loop);
 
   return 1;
 }
@@ -1196,4 +1222,9 @@ void
 bfd_build(void)
 {
   proto_build(&proto_bfd);
+
+  bfd_global.lock = DOMAIN_NEW(rtable, "BFD Global");
+  init_list(&bfd_global.wait_list);
+  init_list(&bfd_global.pickup_list);
+  init_list(&bfd_global.proto_list);
 }

proto/bfd/bfd.h

@@ -17,12 +17,12 @@
 #include "nest/password.h"
 #include "conf/conf.h"
 #include "lib/hash.h"
+#include "lib/io-loop.h"
 #include "lib/resource.h"
 #include "lib/socket.h"
 #include "lib/string.h"
 
 #include "nest/bfd.h"
-#include "io.h"
 
 
 #define BFD_CONTROL_PORT	3784
@@ -88,9 +88,11 @@ struct bfd_neighbor
 struct bfd_proto
 {
   struct proto p;
-  struct birdloop *loop;
+
-  pool *tpool;
   pthread_spinlock_t lock;
+
+  pool *tpool;
+
   node bfd_node;
 
   slab *session_slab;

proto/bfd/config.Y

@@ -37,6 +37,7 @@ proto: bfd_proto ;
 bfd_proto_start: proto_start BFD
 {
   this_proto = proto_config_new(&proto_bfd, $1);
+  this_proto->loop_order = DOMAIN_ORDER(proto);
   init_list(&BFD_CFG->patt_list);
   init_list(&BFD_CFG->neigh_list);
   BFD_CFG->accept_ipv4 = BFD_CFG->accept_ipv6 = 1;

proto/bfd/io.h

@@ -1,34 +0,0 @@
-/*
- *	BIRD -- I/O and event loop
- *
- *	Can be freely distributed and used under the terms of the GNU GPL.
- */
-
-#ifndef _BIRD_BFD_IO_H_
-#define _BIRD_BFD_IO_H_
-
-#include "nest/bird.h"
-#include "lib/lists.h"
-#include "lib/resource.h"
-#include "lib/event.h"
-#include "lib/timer.h"
-#include "lib/socket.h"
-
-
-void ev2_schedule(event *e);
-
-void sk_start(sock *s);
-void sk_stop(sock *s);
-
-struct birdloop *birdloop_new(void);
-void birdloop_start(struct birdloop *loop);
-void birdloop_stop(struct birdloop *loop);
-void birdloop_free(struct birdloop *loop);
-
-void birdloop_enter(struct birdloop *loop);
-void birdloop_leave(struct birdloop *loop);
-void birdloop_mask_wakeups(struct birdloop *loop);
-void birdloop_unmask_wakeups(struct birdloop *loop);
-
-
-#endif /* _BIRD_BFD_IO_H_ */

proto/bfd/packets.c

@@ -412,7 +412,7 @@ bfd_err_hook(sock *sk, int err)
 sock *
 bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af)
 {
-  sock *sk = sk_new(p->tpool);
+  sock *sk = sk_new(p->p.pool);
   sk->type = SK_UDP;
   sk->subtype = af;
   sk->sport = !multihop ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT;
@@ -475,7 +475,7 @@ bfd_open_rx_sk_bound(struct bfd_proto *p, ip_addr local, struct iface *ifa)
 sock *
 bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa)
 {
-  sock *sk = sk_new(p->tpool);
+  sock *sk = sk_new(p->p.pool);
   sk->type = SK_UDP;
   sk->saddr = local;
   sk->dport = ifa ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT;

proto/bgp/bgp.c

@@ -832,6 +832,7 @@ bgp_graceful_restart_feed(struct bgp_channel *c)
 {
   c->stale_feed = (struct rt_export_request) {
     .name = "BGP-GR",
+    .list = &global_work_list,
     .trace_routes = c->c.debug | c->c.proto->debug,
     .dump_req = bgp_graceful_restart_feed_dump_req,
     .log_state_change = bgp_graceful_restart_feed_log_state_change,

sysdep/unix/Makefile

@@ -1,4 +1,4 @@
-src := alloc.c io.c krt.c log.c main.c random.c coroutine.c
+src := alloc.c io.c io-loop.c krt.c log.c main.c random.c coroutine.c
 obj := $(src-o-files)
 $(all-daemon)
 $(cf-local)

sysdep/unix/alloc.c

@@ -97,7 +97,7 @@ alloc_page(void)
     struct free_page *fp = SKIP_BACK(struct free_page, n, HEAD(fps->pages));
     rem_node(&fp->n);
     if ((--fps->cnt < fps->min) && !shutting_down)
-      ev_schedule(&fps->cleanup);
+      ev_send(&global_work_list, &fps->cleanup);
 
     bzero(fp, page_size);
     return fp;
@@ -124,7 +124,7 @@ free_page(void *ptr)
   add_tail(&fps->pages, &fp->n);
 
   if ((++fps->cnt > fps->max) && !shutting_down)
-    ev_schedule(&fps->cleanup);
+    ev_send(&global_work_list, &fps->cleanup);
 #endif
 }
 

sysdep/unix/coroutine.c

@@ -21,10 +21,9 @@
 #include "lib/resource.h"
 #include "lib/timer.h"
 
-/* Using a rather big stack for coroutines to allow for stack-local allocations.
+#include "conf/conf.h"
- * In real world, the kernel doesn't alloc this memory until it is used.
+
- * */
+#define CORO_STACK_SIZE	65536
-#define CORO_STACK_SIZE	1048576
 
 /*
  *	Implementation of coroutines based on POSIX threads
@@ -79,6 +78,11 @@ domain_free(struct domain_generic *dg)
   xfree(dg);
 }
 
+uint dg_order(struct domain_generic *dg)
+{
+  return dg->order;
+}
+
 void do_lock(struct domain_generic *dg, struct domain_generic **lsp)
 {
   if ((char *) lsp - (char *) &locking_stack != dg->order)
@@ -89,7 +93,11 @@ void do_lock(struct domain_generic *dg, struct domain_generic **lsp)
   if (*lsp)
     bug("Inconsistent locking stack state on lock");
 
+  btime lock_begin = current_time();
   pthread_mutex_lock(&dg->mutex);
+  btime duration = current_time() - lock_begin;
+  if (config && (duration > config->watchdog_warning))
+    log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS));
 
   if (dg->prev || dg->locked_by)
     bug("Previous unlock not finished correctly");
@@ -140,11 +148,16 @@ static struct resclass coro_class = {
   .free = coro_free,
 };
 
+_Thread_local struct coroutine *this_coro = NULL;
+
 static void *coro_entry(void *p)
 {
   struct coroutine *c = p;
+
   ASSERT_DIE(c->entry);
 
+  this_coro = c;
+
   c->entry(c->data);
   ASSERT_DIE(coro_cleaned_up);
 

sysdep/unix/io-loop.c

@@ -15,50 +15,47 @@
 #include <sys/time.h>
 
 #include "nest/bird.h"
-#include "proto/bfd/io.h"
 
 #include "lib/buffer.h"
+#include "lib/coro.h"
 #include "lib/lists.h"
 #include "lib/resource.h"
 #include "lib/event.h"
 #include "lib/timer.h"
 #include "lib/socket.h"
 
-
+#include "lib/io-loop.h"
-struct birdloop
+#include "sysdep/unix/io-loop.h"
-{
+#include "conf/conf.h"
-  pool *pool;
-  pthread_t thread;
-  pthread_mutex_t mutex;
-
-  u8 stop_called;
-  u8 poll_active;
-  u8 wakeup_masked;
-  int wakeup_fds[2];
-
-  struct timeloop time;
-  list event_list;
-  list sock_list;
-  uint sock_num;
-
-  BUFFER(sock *) poll_sk;
-  BUFFER(struct pollfd) poll_fd;
-  u8 poll_changed;
-  u8 close_scheduled;
-};
-
 
 /*
  *	Current thread context
  */
 
-static _Thread_local struct birdloop *birdloop_current;
+_Thread_local struct birdloop *birdloop_current;
+static _Thread_local struct birdloop *birdloop_wakeup_masked;
+static _Thread_local uint birdloop_wakeup_masked_count;
 
-static inline void
+event_list *
-birdloop_set_current(struct birdloop *loop)
+birdloop_event_list(struct birdloop *loop)
 {
-  birdloop_current = loop;
+  return &loop->event_list;
-  local_timeloop = loop ? &loop->time : &main_timeloop;
+}
+
+struct timeloop *
+birdloop_time_loop(struct birdloop *loop)
+{
+  return &loop->time;
+}
+
+_Bool
+birdloop_inside(struct birdloop *loop)
+{
+  for (struct birdloop *c = birdloop_current; c; c = c->prev_loop)
+    if (loop == c)
+      return 1;
+
+  return 0;
 }
 
 /*
@@ -135,57 +132,17 @@ wakeup_do_kick(struct birdloop *loop)
   pipe_kick(loop->wakeup_fds[1]);
 }
 
-static inline void
+void
-wakeup_kick(struct birdloop *loop)
+birdloop_ping(struct birdloop *loop)
 {
-  if (!loop->wakeup_masked)
+  u32 ping_sent = atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel);
-    wakeup_do_kick(loop);
+  if (ping_sent)
+    return;
+
+  if (loop == birdloop_wakeup_masked)
+    birdloop_wakeup_masked_count++;
   else
-    loop->wakeup_masked = 2;
+    wakeup_do_kick(loop);
-}
-
-/* For notifications from outside */
-void
-wakeup_kick_current(void)
-{
-  if (birdloop_current && birdloop_current->poll_active)
-    wakeup_kick(birdloop_current);
-}
-
-
-/*
- *	Events
- */
-
-static inline uint
-events_waiting(struct birdloop *loop)
-{
-  return !EMPTY_LIST(loop->event_list);
-}
-
-static inline void
-events_init(struct birdloop *loop)
-{
-  init_list(&loop->event_list);
-}
-
-static void
-events_fire(struct birdloop *loop)
-{
-  times_update();
-  ev_run_list(&loop->event_list);
-}
-
-void
-ev2_schedule(event *e)
-{
-  if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list))
-    wakeup_kick(birdloop_current);
-
-  if (e->n.next)
-    rem_node(&e->n);
-
-  add_tail(&birdloop_current->event_list, &e->n);
 }
 
 
@@ -213,13 +170,13 @@ sockets_add(struct birdloop *loop, sock *s)
   s->index = -1;
   loop->poll_changed = 1;
 
-  if (loop->poll_active)
+  birdloop_ping(loop);
-    wakeup_kick(loop);
 }
 
 void
 sk_start(sock *s)
 {
+  ASSERT_DIE(birdloop_current != &main_birdloop);
   sockets_add(birdloop_current, s);
 }
 
@@ -230,28 +187,21 @@ sockets_remove(struct birdloop *loop, sock *s)
   loop->sock_num--;
 
   if (s->index >= 0)
+  {
     loop->poll_sk.data[s->index] = NULL;
-
+    s->index = -1;
-  s->index = -1;
+    loop->poll_changed = 1;
-  loop->poll_changed = 1;
+    loop->close_scheduled = 1;
-
+    birdloop_ping(loop);
-  /* Wakeup moved to sk_stop() */
+  }
+  else
+    close(s->fd);
 }
 
 void
 sk_stop(sock *s)
 {
   sockets_remove(birdloop_current, s);
-
-  if (birdloop_current->poll_active)
-  {
-    birdloop_current->close_scheduled = 1;
-    wakeup_kick(birdloop_current);
-  }
-  else
-    close(s->fd);
-
-  s->fd = -1;
 }
 
 static inline uint sk_want_events(sock *s)
@@ -351,12 +301,15 @@ sockets_fire(struct birdloop *loop)
 
     if (pfd->revents & POLLIN)
       while (e && *psk && (*psk)->rx_hook)
-	e = sk_read(*psk, 0);
+	e = sk_read(*psk, pfd->revents);
 
     e = 1;
     if (pfd->revents & POLLOUT)
+    {
+      loop->poll_changed = 1;
       while (e && *psk)
 	e = sk_write(*psk);
+    }
   }
 }
 
@@ -365,106 +318,170 @@ sockets_fire(struct birdloop *loop)
  *	Birdloop
  */
 
-static void *birdloop_main(void *arg);
+struct birdloop main_birdloop;
+
+static void birdloop_enter_locked(struct birdloop *loop);
+
+void
+birdloop_init(void)
+{
+  wakeup_init(&main_birdloop);
+
+  main_birdloop.time.domain = the_bird_domain.the_bird;
+  main_birdloop.time.loop = &main_birdloop;
+
+  times_update();
+  timers_init(&main_birdloop.time, &root_pool);
+
+  birdloop_enter_locked(&main_birdloop);
+}
+
+static void birdloop_main(void *arg);
 
 struct birdloop *
-birdloop_new(void)
+birdloop_new(pool *pp, uint order, const char *name)
 {
-  pool *p = rp_new(NULL, "Birdloop root");
+  struct domain_generic *dg = domain_new(name, order);
+
+  pool *p = rp_new(pp, name);
   struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop));
   loop->pool = p;
-  pthread_mutex_init(&loop->mutex, NULL);
+
+  loop->time.domain = dg;
+  loop->time.loop = loop;
+
+  birdloop_enter(loop);
 
   wakeup_init(loop);
-
+  ev_init_list(&loop->event_list, loop, name);
-  events_init(loop);
   timers_init(&loop->time, p);
   sockets_init(loop);
 
+  loop->time.coro = coro_run(p, birdloop_main, loop);
+
+  birdloop_leave(loop);
+
   return loop;
 }
 
-void
+static void
-birdloop_start(struct birdloop *loop)
+birdloop_do_stop(struct birdloop *loop, void (*stopped)(void *data), void *data)
 {
-  int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop);
+  loop->stopped = stopped;
-  if (rv)
+  loop->stop_data = data;
-    die("pthread_create(): %M", rv);
+  wakeup_do_kick(loop);
 }
 
 void
-birdloop_stop(struct birdloop *loop)
+birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data)
 {
-  pthread_mutex_lock(&loop->mutex);
+  DG_LOCK(loop->time.domain);
-  loop->stop_called = 1;
+  birdloop_do_stop(loop, stopped, data);
-  wakeup_do_kick(loop);
+  DG_UNLOCK(loop->time.domain);
-  pthread_mutex_unlock(&loop->mutex);
+}
 
-  int rv = pthread_join(loop->thread, NULL);
+void
-  if (rv)
+birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data)
-    die("pthread_join(): %M", rv);
+{
+  ASSERT_DIE(loop == birdloop_current);
+  ASSERT_DIE(DG_IS_LOCKED(loop->time.domain));
+
+  birdloop_do_stop(loop, stopped, data);
 }
 
 void
 birdloop_free(struct birdloop *loop)
 {
+  ASSERT_DIE(loop->links == 0);
+  domain_free(loop->time.domain);
   rfree(loop->pool);
 }
 
+static void
+birdloop_enter_locked(struct birdloop *loop)
+{
+  ASSERT_DIE(DG_IS_LOCKED(loop->time.domain));
+  ASSERT_DIE(!birdloop_inside(loop));
+
+  /* Store the old context */
+  loop->prev_loop = birdloop_current;
+
+  /* Put the new context */
+  birdloop_current = loop;
+}
 
 void
 birdloop_enter(struct birdloop *loop)
 {
-  /* TODO: these functions could save and restore old context */
+  DG_LOCK(loop->time.domain);
-  pthread_mutex_lock(&loop->mutex);
+  return birdloop_enter_locked(loop);
-  birdloop_set_current(loop);
+}
+
+static void
+birdloop_leave_locked(struct birdloop *loop)
+{
+  /* Check the current context */
+  ASSERT_DIE(birdloop_current == loop);
+
+  /* Restore the old context */
+  birdloop_current = loop->prev_loop;
 }
 
 void
 birdloop_leave(struct birdloop *loop)
 {
-  /* TODO: these functions could save and restore old context */
+  birdloop_leave_locked(loop);
-  birdloop_set_current(NULL);
+  DG_UNLOCK(loop->time.domain);
-  pthread_mutex_unlock(&loop->mutex);
 }
 
 void
 birdloop_mask_wakeups(struct birdloop *loop)
 {
-  pthread_mutex_lock(&loop->mutex);
+  ASSERT_DIE(birdloop_wakeup_masked == NULL);
-  loop->wakeup_masked = 1;
+  birdloop_wakeup_masked = loop;
-  pthread_mutex_unlock(&loop->mutex);
 }
 
 void
 birdloop_unmask_wakeups(struct birdloop *loop)
 {
-  pthread_mutex_lock(&loop->mutex);
+  ASSERT_DIE(birdloop_wakeup_masked == loop);
-  if (loop->wakeup_masked == 2)
+  birdloop_wakeup_masked = NULL;
+  if (birdloop_wakeup_masked_count)
     wakeup_do_kick(loop);
-  loop->wakeup_masked = 0;
+
-  pthread_mutex_unlock(&loop->mutex);
+  birdloop_wakeup_masked_count = 0;
 }
 
-static void *
+void
+birdloop_link(struct birdloop *loop)
+{
+  ASSERT_DIE(birdloop_inside(loop));
+  loop->links++;
+}
+
+void
+birdloop_unlink(struct birdloop *loop)
+{
+  ASSERT_DIE(birdloop_inside(loop));
+  loop->links--;
+}
+
+static void
 birdloop_main(void *arg)
 {
   struct birdloop *loop = arg;
   timer *t;
   int rv, timeout;
 
-  birdloop_set_current(loop);
+  btime loop_begin = current_time();
 
   tmp_init(loop->pool);
 
-  pthread_mutex_lock(&loop->mutex);
+  birdloop_enter(loop);
   while (1)
   {
-    events_fire(loop);
+    timers_fire(&loop->time, 0);
-    timers_fire(&loop->time);
+    if (ev_run_list(&loop->event_list))
-
-    times_update();
-    if (events_waiting(loop))
       timeout = 0;
     else if (t = timers_first(&loop->time))
       timeout = (tm_remains(t) TO_MS) + 1;
@@ -474,8 +491,11 @@ birdloop_main(void *arg)
     if (loop->poll_changed)
       sockets_prepare(loop);
 
-    loop->poll_active = 1;
+    btime duration = current_time() - loop_begin;
-    pthread_mutex_unlock(&loop->mutex);
+    if (duration > config->watchdog_warning)
+      log(L_WARN "I/O loop cycle took %d ms", (int) (duration TO_MS));
+
+    birdloop_leave(loop);
 
   try:
     rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout);
@@ -486,25 +506,32 @@ birdloop_main(void *arg)
       die("poll: %m");
     }
 
-    pthread_mutex_lock(&loop->mutex);
+    birdloop_enter(loop);
-    loop->poll_active = 0;
 
     if (loop->close_scheduled)
       sockets_close_fds(loop);
 
-    if (loop->stop_called)
+    if (loop->stopped)
       break;
 
+    loop_begin = current_time();
+
     if (rv)
       sockets_fire(loop);
 
-    timers_fire(&loop->time);
+    atomic_exchange_explicit(&loop->ping_sent, 0, memory_order_acq_rel);
   }
 
-  loop->stop_called = 0;
+  /* Flush remaining events */
-  pthread_mutex_unlock(&loop->mutex);
+  ASSERT_DIE(!ev_run_list(&loop->event_list));
 
-  return NULL;
+  /* No timers allowed */
+  ASSERT_DIE(timers_count(&loop->time) == 0);
+  ASSERT_DIE(EMPTY_LIST(loop->sock_list));
+  ASSERT_DIE(loop->sock_num == 0);
+
+  birdloop_leave(loop);
+  loop->stopped(loop->stop_data);
 }
 
 

sysdep/unix/io-loop.h

@@ -0,0 +1,35 @@
+/*
+ *	BIRD -- I/O and event loop
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL.
+ */
+
+#ifndef _BIRD_SYSDEP_UNIX_IO_LOOP_H_
+#define _BIRD_SYSDEP_UNIX_IO_LOOP_H_
+
+struct birdloop
+{
+  pool *pool;
+
+  struct timeloop time;
+  event_list event_list;
+  list sock_list;
+  uint sock_num;
+
+  BUFFER(sock *) poll_sk;
+  BUFFER(struct pollfd) poll_fd;
+  u8 poll_changed;
+  u8 close_scheduled;
+
+  _Atomic u32 ping_sent;
+  int wakeup_fds[2];
+
+  uint links;
+
+  void (*stopped)(void *data);
+  void *stop_data;
+
+  struct birdloop *prev_loop;
+};
+
+#endif

sysdep/unix/io.c

@@ -43,6 +43,7 @@
 #include "conf/conf.h"
 
 #include "sysdep/unix/unix.h"
+#include "sysdep/unix/io-loop.h"
 #include CONFIG_INCLUDE_SYSIO_H
 
 /* Maximum number of calls of tx handler for one socket in one
@@ -800,18 +801,16 @@ sk_free(resource *r)
     sk_ssh_free(s);
 #endif
 
-  if (s->fd < 0)
+  if ((s->fd < 0) || (s->flags & SKF_THREAD))
     return;
 
-  /* FIXME: we should call sk_stop() for SKF_THREAD sockets */
+  if (s == current_sock)
-  if (!(s->flags & SKF_THREAD))
+    current_sock = sk_next(s);
-  {
+  if (s == stored_sock)
-    if (s == current_sock)
+    stored_sock = sk_next(s);
-      current_sock = sk_next(s);
+
-    if (s == stored_sock)
+  if (enlisted(&s->n))
-      stored_sock = sk_next(s);
     rem_node(&s->n);
-  }
 
   if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE)
     close(s->fd);
@@ -1104,7 +1103,11 @@ sk_passive_connected(sock *s, int type)
     return 1;
   }
 
-  sk_insert(t);
+  if (s->flags & SKF_PASSIVE_THREAD)
+    t->flags |= SKF_THREAD;
+  else
+    sk_insert(t);
+
   sk_alloc_bufs(t);
   s->rx_hook(t, 0);
   return 1;
@@ -1512,6 +1515,36 @@ sk_open_unix(sock *s, char *name)
   return 0;
 }
 
+static void
+sk_reloop_hook(void *_vs)
+{
+  sock *s = _vs;
+  if (birdloop_inside(&main_birdloop))
+  {
+    s->flags &= ~SKF_THREAD;
+    sk_insert(s);
+  }
+  else
+  {
+    s->flags |= SKF_THREAD;
+    sk_start(s);
+  }
+}
+
+void
+sk_reloop(sock *s, struct birdloop *loop)
+{
+  if (enlisted(&s->n))
+    rem_node(&s->n);
+
+  s->reloop = (event) {
+    .hook = sk_reloop_hook,
+    .data = s,
+  };
+
+  ev_send_loop(loop, &s->reloop);
+}
+
 
 #define CMSG_RX_SPACE MAX(CMSG4_SPACE_PKTINFO+CMSG4_SPACE_TTL, \
 			  CMSG6_SPACE_PKTINFO+CMSG6_SPACE_TTL)
@@ -2164,8 +2197,9 @@ void
 io_init(void)
 {
   init_list(&sock_list);
-  init_list(&global_event_list);
+  ev_init_list(&global_event_list, &main_birdloop, "Global event list");
-  init_list(&global_work_list);
+  ev_init_list(&global_work_list, &main_birdloop, "Global work list");
+  ev_init_list(&main_birdloop.event_list, &main_birdloop, "Global fast event list");
   krt_io_init();
   // XXX init_times();
   // XXX update_times();
@@ -2179,14 +2213,7 @@ static int short_loops = 0;
 #define SHORT_LOOP_MAX 10
 #define WORK_EVENTS_MAX 10
 
-static int poll_reload_pipe[2];
+void pipe_drain(int fd);
-
-void
-io_loop_reload(void)
-{
-  char b;
-  write(poll_reload_pipe[1], &b, 1);
-}
 
 void
 io_loop(void)
@@ -2199,21 +2226,19 @@ io_loop(void)
   int fdmax = 256;
   struct pollfd *pfd = xmalloc(fdmax * sizeof(struct pollfd));
 
-  if (pipe(poll_reload_pipe) < 0)
-    die("pipe(poll_reload_pipe) failed: %m");
-
   watchdog_start1();
   for(;;)
     {
       times_update();
       events = ev_run_list(&global_event_list);
       events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events;
-      timers_fire(&main_timeloop);
+      events = ev_run_list(&main_birdloop.event_list) || events;
+      timers_fire(&main_birdloop.time, 1);
       io_close_event();
 
       // FIXME
       poll_tout = (events ? 0 : 3000); /* Time in milliseconds */
-      if (t = timers_first(&main_timeloop))
+      if (t = timers_first(&main_birdloop.time))
       {
 	times_update();
 	timeout = (tm_remains(t) TO_MS) + 1;
@@ -2221,7 +2246,7 @@ io_loop(void)
       }
 
       /* A hack to reload main io_loop() when something has changed asynchronously. */
-      pfd[0].fd = poll_reload_pipe[0];
+      pfd[0].fd = main_birdloop.wakeup_fds[0];
       pfd[0].events = POLLIN;
 
       nfds = 1;
@@ -2284,9 +2309,9 @@ io_loop(void)
 
       /* And finally enter poll() to find active sockets */
       watchdog_stop();
-      the_bird_unlock();
+      birdloop_leave(&main_birdloop);
       pout = poll(pfd, nfds, poll_tout);
-      the_bird_lock();
+      birdloop_enter(&main_birdloop);
       watchdog_start();
 
       if (pout < 0)
@@ -2300,8 +2325,8 @@ io_loop(void)
 	  if (pfd[0].revents & POLLIN)
 	  {
 	    /* IO loop reload requested */
-	    char b;
+	    pipe_drain(main_birdloop.wakeup_fds[0]);
-	    read(poll_reload_pipe[0], &b, 1);
+	    atomic_exchange_explicit(&main_birdloop.ping_sent, 0, memory_order_acq_rel);
 	    continue;
 	  }
 

sysdep/unix/main.c

@@ -878,10 +878,12 @@ main(int argc, char **argv)
   parse_args(argc, argv);
   log_switch(1, NULL, NULL);
 
+  the_bird_lock();
+
   random_init();
   net_init();
   resource_init();
-  timer_init();
+  birdloop_init();
   olock_init();
   rt_init();
   io_init();
@@ -929,7 +931,6 @@ main(int argc, char **argv)
       dup2(0, 2);
     }
 
-  the_bird_lock();
 
   main_thread_init();
 

sysdep/unix/unix.h

@@ -106,7 +106,6 @@ extern volatile sig_atomic_t async_shutdown_flag;
 
 void io_init(void);
 void io_loop(void);
-void io_loop_reload(void);
 void io_log_dump(void);
 int sk_open_unix(struct birdsock *s, char *name);
 struct rfile *rf_open(struct pool *, const char *name, const char *mode);

test/birdtest.c

@@ -21,6 +21,7 @@
 #include "test/birdtest.h"
 #include "lib/string.h"
 #include "lib/event.h"
+#include "lib/io-loop.h"
 
 #ifdef HAVE_EXECINFO_H
 #include <execinfo.h>
@@ -121,8 +122,8 @@ bt_init(int argc, char *argv[])
   bt_suite_case_begin = bt_suite_begin = bt_begin;
 
   resource_init();
-  ev_init_list(&global_event_list);
+  ev_init_list(&global_event_list, &main_birdloop, "Global event list in unit tests");
-
+  ev_init_list(&global_work_list, &main_birdloop, "Global work list in unit tests");
   return;
 
  usage:

test/bt-utils.c

@@ -53,6 +53,8 @@ cf_file_read(byte *dest, uint max_len, int fd)
   return l;
 }
 
+void resource_sys_init(void);
+
 void
 bt_bird_init(void)
 {
@@ -60,8 +62,9 @@ bt_bird_init(void)
     log_init_debug("");
   log_switch(bt_verbose != 0, NULL, NULL);
 
+  the_bird_lock();
   olock_init();
-  timer_init();
+  birdloop_init();
   rt_init();
   io_init();
   if_init();
@@ -73,6 +76,7 @@ bt_bird_init(void)
 void bt_bird_cleanup(void)
 {
   config = new_config = NULL;
+  the_bird_unlock();
 }
 
 static char *