Converting the former BFD loop to a universal IO loop and protocol loop.

There is a simple universal IO loop, taking care of events, timers and
sockets. Primarily, one instance of a protocol should use exactly one IO
loop to do all its work, as is now done in BFD.

Contrary to previous versions, the loop is now launched and cleaned by
the nest/proto.c code, allowing for a protocol to just request its own
loop by setting the loop's lock order in config higher than the_bird.

It is not supported nor checked if any protocol changed the requested
lock order in reconfigure. No protocol should do it at all.

filter/filter_test.c

@@ -72,6 +72,7 @@ int
 main(int argc, char *argv[])
 {
   bt_init(argc, argv);
+
   bt_bird_init();
 
   bt_assert_hook = bt_assert_filter;

lib/birdlib.h

@@ -71,6 +71,7 @@ static inline int u64_cmp(u64 i1, u64 i2)
 /* Macros for gcc attributes */
 
 #define NORET __attribute__((noreturn))
+#define USE_RESULT __atribute__((warn_unused_result))
 #define UNUSED __attribute__((unused))
 #define PACKED __attribute__((packed))
 #define NONNULL(...) __attribute__((nonnull((__VA_ARGS__))))

lib/coro.h

@@ -2,7 +2,7 @@
  *	BIRD Coroutines
  *
  *	(c) 2017 Martin Mares <mj@ucw.cz>
- *	(c) 2020 Maria Matejka <mq@jmq.cz>
+ *	(c) 2020-2021 Maria Matejka <mq@jmq.cz>
  *
  *	Can be freely distributed and used under the terms of the GNU GPL.
  */
@@ -22,5 +22,8 @@ struct coroutine;
  */
 struct coroutine *coro_run(pool *, void (*entry)(void *), void *data);
 
+/* Get self. */
+extern _Thread_local struct coroutine *this_coro;
+
 
 #endif

lib/event.c

@@ -19,8 +19,14 @@
  * events in them and explicitly ask to run them.
  */
 
+#undef LOCAL_DEBUG
+
 #include "nest/bird.h"
 #include "lib/event.h"
+#include "lib/locking.h"
+#include "lib/io-loop.h"
+
+extern _Thread_local struct coroutine *this_coro;
 
 event_list global_event_list;
 event_list global_work_list;
@@ -28,11 +34,16 @@ event_list global_work_list;
 inline void
 ev_postpone(event *e)
 {
+  event_list *el = e->list;
+  if (!el)
+    return;
+
+  ASSERT_DIE(birdloop_inside(el->loop));
+
+  LOCK_DOMAIN(event, el->lock);
   if (ev_active(e))
-    {
     rem_node(&e->n);
-      e->n.next = NULL;
+  UNLOCK_DOMAIN(event, el->lock);
-    }
 }
 
 static void
@@ -95,40 +106,25 @@ ev_run(event *e)
  * list @l which can be run by calling ev_run_list().
  */
 inline void
-ev_enqueue(event_list *l, event *e)
+ev_send(event_list *l, event *e)
 {
-  ev_postpone(e);
+  DBG("ev_send(%p, %p)\n", l, e);
-  add_tail(l, &e->n);
+  ASSERT_DIE(e->hook);
+  ASSERT_DIE(!e->list || (e->list == l) || (e->list->loop == l->loop));
+
+  e->list = l;
+
+  LOCK_DOMAIN(event, l->lock);
+  if (enlisted(&e->n))
+  {
+    UNLOCK_DOMAIN(event, l->lock);
+    return;
   }
 
-/**
+  add_tail(&l->events, &e->n);
- * ev_schedule - schedule an event
+  UNLOCK_DOMAIN(event, l->lock);
- * @e: an event
- *
- * This function schedules an event by enqueueing it to a system-wide
- * event list which is run by the platform dependent code whenever
- * appropriate.
- */
-void
-ev_schedule(event *e)
-{
-  ev_enqueue(&global_event_list, e);
-}
 
-/**
+  birdloop_ping(l->loop);
- * ev_schedule_work - schedule a work-event.
- * @e: an event
- *
- * This function schedules an event by enqueueing it to a system-wide work-event
- * list which is run by the platform dependent code whenever appropriate. This
- * is designated for work-events instead of regular events. They are executed
- * less often in order to not clog I/O loop.
- */
-void
-ev_schedule_work(event *e)
-{
-  if (!ev_active(e))
-    add_tail(&global_work_list, &e->n);
 }
 
 void io_log_event(void *hook, void *data);
@@ -142,35 +138,64 @@ void io_log_event(void *hook, void *data);
 int
 ev_run_list(event_list *l)
 {
-  node *n;
+  const _Bool legacy = LEGACY_EVENT_LIST(l);
-  list tmp_list;
 
+  if (legacy)
+    ASSERT_THE_BIRD_LOCKED;
+
+  node *n;
+
+  list tmp_list;
   init_list(&tmp_list);
-  add_tail_list(&tmp_list, l);
+
-  init_list(l);
+  /* Move the event list contents to a local list to avoid executing repeatedly added events */
+  LOCK_DOMAIN(event, l->lock);
+  add_tail_list(&tmp_list, &l->events);
+  init_list(&l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+
   WALK_LIST_FIRST(n, tmp_list)
     {
       event *e = SKIP_BACK(event, n, n);
 
-      /* This is ugly hack, we want to log just events executed from the main I/O loop */
+      if (legacy)
-      if ((l == &global_event_list) || (l == &global_work_list))
+      {
+	/* The legacy way of event execution */
 	io_log_event(e->hook, e->data);
-
+	ev_postpone(e);
-      ev_run(e);
+	e->hook(e->data);
+      }
+      else
+      {
+	// io_log_event(e->hook, e->data); /* TODO: add support for event logging in other io loops */
+	ASSERT_DIE(e->list == l);
+	LOCK_DOMAIN(event, l->lock);
+	rem_node(&e->n);
+	UNLOCK_DOMAIN(event, l->lock);
+	e->hook(e->data);
+      }
     }
 
-  return !EMPTY_LIST(*l);
+  LOCK_DOMAIN(event, l->lock);
+  int repeat = ! EMPTY_LIST(l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+  return repeat;
 }
 
 int
 ev_run_list_limited(event_list *l, uint limit)
 {
+  ASSERT_DIE(LEGACY_EVENT_LIST(l));
+  ASSERT_THE_BIRD_LOCKED;
+
   node *n;
   list tmp_list;
 
+  LOCK_DOMAIN(event, l->lock);
   init_list(&tmp_list);
-  add_tail_list(&tmp_list, l);
+  add_tail_list(&tmp_list, &l->events);
-  init_list(l);
+  init_list(&l->events);
+  UNLOCK_DOMAIN(event, l->lock);
 
   WALK_LIST_FIRST(n, tmp_list)
     {
@@ -179,21 +204,23 @@ ev_run_list_limited(event_list *l, uint limit)
       if (!limit)
 	break;
 
-      /* This is ugly hack, we want to log just events executed from the main I/O loop */
-      if ((l == &global_event_list) || (l == &global_work_list))
       io_log_event(e->hook, e->data);
 
       ev_run(e);
       limit--;
     }
 
+  LOCK_DOMAIN(event, l->lock);
   if (!EMPTY_LIST(tmp_list))
   {
     /* Attach new items after the unprocessed old items */
-    add_tail_list(&tmp_list, l);
+    add_tail_list(&tmp_list, &l->events);
-    init_list(l);
+    init_list(&l->events);
-    add_tail_list(l, &tmp_list);
+    add_tail_list(&l->events, &tmp_list);
   }
 
-  return !EMPTY_LIST(*l);
+  int repeat = ! EMPTY_LIST(l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+
+  return repeat;
 }

lib/event.h

@@ -10,33 +10,62 @@
 #define _BIRD_EVENT_H_
 
 #include "lib/resource.h"
+#include "lib/locking.h"
+
+#include <stdatomic.h>
+
+DEFINE_DOMAIN(event);
 
 typedef struct event {
   resource r;
   void (*hook)(void *);
   void *data;
   node n;				/* Internal link */
+  struct event_list *list;		/* List where this event is put in */
 } event;
 
-typedef list event_list;
+typedef struct event_list {
+  list events;
+  pool *pool;
+  struct birdloop *loop;
+  DOMAIN(event) lock;
+} event_list;
 
 extern event_list global_event_list;
 extern event_list global_work_list;
 
 event *ev_new(pool *);
 void ev_run(event *);
-#define ev_init_list(el) init_list(el)
+
-void ev_enqueue(event_list *, event *);
+static inline void ev_init_list(event_list *el, struct birdloop *loop, const char *name)
-void ev_schedule(event *);
+{
-void ev_schedule_work(event *);
+  init_list(&el->events);
+  el->loop = loop;
+  el->lock = DOMAIN_NEW(event, name);
+}
+
+void ev_send(event_list *, event *);
+#define ev_send_loop(l, e) ev_send(birdloop_event_list((l)), (e))
+
+#define ev_schedule(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_event_list, (e)); })
+#define ev_schedule_work(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_work_list, (e)); })
+
 void ev_postpone(event *);
 int ev_run_list(event_list *);
 int ev_run_list_limited(event_list *, uint);
 
+#define LEGACY_EVENT_LIST(l)  (((l) == &global_event_list) || ((l) == &global_work_list))
+
+_Bool birdloop_inside(struct birdloop *loop);
+
 static inline int
 ev_active(event *e)
 {
-  return e->n.next != NULL;
+  if (e->list == NULL)
+    return 0;
+
+  ASSERT_DIE(birdloop_inside(e->list->loop));
+  return enlisted(&e->n);
 }
 
 static inline event*

lib/event_test.c

@@ -48,14 +48,17 @@ init_event_check_points(void)
     event_check_points[i] = 0;
 }
 
+void resource_sys_init(void);
+
 static int
 t_ev_run_list(void)
 {
   int i;
 
+  resource_sys_init();
   resource_init();
   olock_init();
-  timer_init();
+  birdloop_init();
   io_init();
   rt_init();
   if_init();
@@ -82,7 +85,9 @@ main(int argc, char *argv[])
 {
   bt_init(argc, argv);
 
+  the_bird_lock();
   bt_test_suite(t_ev_run_list, "Schedule and run 3 events in right order.");
+  the_bird_unlock();
 
   return bt_exit_value();
 }

lib/flowspec_test.c

@@ -666,10 +666,13 @@ t_formatting6(void)
   return 1;
 }
 
+void resource_sys_init(void);
+
 int
 main(int argc, char *argv[])
 {
   bt_init(argc, argv);
+  resource_sys_init();
 
   bt_test_suite(t_read_length,  "Testing get NLRI length");
   bt_test_suite(t_write_length, "Testing set NLRI length");

lib/io-loop.h

@@ -0,0 +1,54 @@
+/*
+ *	BIRD -- I/O and event loop
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL.
+ */
+
+#ifndef _BIRD_IO_LOOP_H_
+#define _BIRD_IO_LOOP_H_
+
+#include "nest/bird.h"
+#include "lib/lists.h"
+#include "lib/locking.h"
+#include "lib/resource.h"
+#include "lib/event.h"
+#include "lib/socket.h"
+
+void sk_start(sock *s);
+void sk_stop(sock *s);
+void sk_reloop(sock *s, struct birdloop *loop);
+
+extern struct birdloop main_birdloop;
+
+/* Start a new birdloop owned by given pool and domain */
+struct birdloop *birdloop_new(pool *p, uint order, const char *name);
+
+/* Stop the loop. At the end, the @stopped callback is called unlocked in tail
+ * position to finish cleanup. Run birdloop_free() from that callback to free
+ * the loop itself. */
+void birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data);
+void birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data);
+void birdloop_free(struct birdloop *loop);
+
+/* Get birdloop's event list */
+event_list *birdloop_event_list(struct birdloop *loop);
+
+/* Get birdloop's time heap */
+struct timeloop *birdloop_time_loop(struct birdloop *loop);
+
+/* Enter and exit the birdloop */
+void birdloop_enter(struct birdloop *loop);
+void birdloop_leave(struct birdloop *loop);
+
+_Bool birdloop_inside(struct birdloop *loop);
+
+void birdloop_mask_wakeups(struct birdloop *loop);
+void birdloop_unmask_wakeups(struct birdloop *loop);
+
+void birdloop_link(struct birdloop *loop);
+void birdloop_unlink(struct birdloop *loop);
+
+void birdloop_ping(struct birdloop *loop);
+
+void birdloop_init(void);
+#endif /* _BIRD_IO_LOOP_H_ */

lib/lists.h

@@ -68,6 +68,18 @@ typedef union list {			/* In fact two overlayed nodes */
 
 #define EMPTY_LIST(list) (!(list).head->next)
 
+static inline _Bool
+enlisted(node *n)
+{
+  switch ((!!n->next) + (!!n->prev))
+  {
+    case 0: return 0;
+    case 2: return 1;
+    case 1: bug("Garbled event list node");
+  }
+
+  bug("Maths is broken. And you should see a new heaven and a new earth: for the first heaven and the first earth had been passed away.");
+}
 
 #ifndef _BIRD_LISTS_C_
 #define LIST_INLINE static inline

lib/locking.h

@@ -14,6 +14,9 @@ struct domain_generic;
 /* Here define the global lock order; first to last. */
 struct lock_order {
   struct domain_generic *the_bird;
+  struct domain_generic *proto;
+  struct domain_generic *rtable;
+  struct domain_generic *event;
 };
 
 extern _Thread_local struct lock_order locking_stack;
@@ -21,24 +24,40 @@ extern _Thread_local struct domain_generic **last_locked;
 
 #define DOMAIN(type) struct domain__##type
 #define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; }
+#define DOMAIN_ORDER(type)  OFFSETOF(struct lock_order, type)
 
-#define DOMAIN_NEW(type, name)  (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) }
+#define DOMAIN_NEW(type, name)  (DOMAIN(type)) { .type = domain_new(name, DOMAIN_ORDER(type)) }
 struct domain_generic *domain_new(const char *name, uint order);
 
+#define DOMAIN_FREE(type, d)	domain_free((d).type)
+void domain_free(struct domain_generic *);
+
 #define DOMAIN_NULL(type)   (DOMAIN(type)) {}
 
 #define LOCK_DOMAIN(type, d)	do_lock(((d).type), &(locking_stack.type))
 #define UNLOCK_DOMAIN(type, d)  do_unlock(((d).type), &(locking_stack.type))
 
+#define DOMAIN_IS_LOCKED(type, d) (((d).type) == (locking_stack.type))
+#define DG_IS_LOCKED(d)	((d) == *(DG_LSP(d)))
+
 /* Internal for locking */
 void do_lock(struct domain_generic *dg, struct domain_generic **lsp);
 void do_unlock(struct domain_generic *dg, struct domain_generic **lsp);
 
+uint dg_order(struct domain_generic *dg);
+
+#define DG_LSP(d)	((struct domain_generic **) (((void *) &locking_stack) + dg_order(d)))
+#define DG_LOCK(d)	do_lock(d, DG_LSP(d))
+#define DG_UNLOCK(d)	do_unlock(d, DG_LSP(d))
+
 /* Use with care. To be removed in near future. */
 DEFINE_DOMAIN(the_bird);
 extern DOMAIN(the_bird) the_bird_domain;
 
 #define the_bird_lock()		LOCK_DOMAIN(the_bird, the_bird_domain)
 #define the_bird_unlock()	UNLOCK_DOMAIN(the_bird, the_bird_domain)
+#define the_bird_locked()	DOMAIN_IS_LOCKED(the_bird, the_bird_domain)
+
+#define ASSERT_THE_BIRD_LOCKED	({ if (!the_bird_locked()) bug("The BIRD lock must be locked here: %s:%d", __FILE__, __LINE__); })
 
 #endif

lib/socket.h

@@ -12,6 +12,7 @@
 #include <errno.h>
 
 #include "lib/resource.h"
+#include "lib/event.h"
 #ifdef HAVE_LIBSSH
 #define LIBSSH_LEGACY_0_4
 #include <libssh/libssh.h>
@@ -79,6 +80,7 @@ typedef struct birdsock {
   const char *password;			/* Password for MD5 authentication */
   const char *err;			/* Error message */
   struct ssh_sock *ssh;			/* Used in SK_SSH */
+  struct event reloop;			/* Reloop event */
 } sock;
 
 sock *sock_new(pool *);			/* Allocate new socket */
@@ -128,6 +130,7 @@ extern int sk_priority_control;		/* Suggested priority for control traffic, shou
 #define SKF_TRUNCATED	0x200	/* Received packet was truncated, set by IO layer */
 #define SKF_HDRINCL	0x400	/* Used internally */
 #define SKF_PKTINFO	0x800	/* Used internally */
+#define SKF_PASSIVE_THREAD  0x1000  /* Child sockets used in thread, do not add to main loop */
 
 /*
  *	Socket types		     SA SP DA DP IF  TTL SendTo	(?=may, -=must not, *=must)

lib/timer.c

@@ -37,15 +37,8 @@
 #include "lib/resource.h"
 #include "lib/timer.h"
 
-
-struct timeloop main_timeloop;
-
-
 #include <pthread.h>
 
-/* Data accessed and modified from proto/bfd/io.c */
-_Thread_local struct timeloop *local_timeloop;
-
 _Atomic btime last_time;
 _Atomic btime real_time;
 
@@ -76,7 +69,7 @@ tm_dump(resource *r)
   if (t->recurrent)
     debug("recur %d, ", t->recurrent);
   if (t->expires)
-    debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS);
+    debug("in loop %p expires in %d ms)\n", t->loop, (t->expires - current_time()) TO_MS);
   else
     debug("inactive)\n");
 }
@@ -99,8 +92,8 @@ tm_new(pool *p)
   return t;
 }
 
-void
+static void
-tm_set(timer *t, btime when)
+tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop)
 {
   uint tc = timers_count(local_timeloop);
 
@@ -122,17 +115,17 @@ tm_set(timer *t, btime when)
     HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
   }
 
-#ifdef CONFIG_BFD
+  t->loop = local_timeloop;
-  /* Hack to notify BFD loops */
+
-  if ((local_timeloop != &main_timeloop) && (t->index == 1))
+  if ((t->index == 1) && (local_timeloop->coro != this_coro))
-    wakeup_kick_current();
+    birdloop_ping(local_timeloop->loop);
-#endif
 }
 
 void
-tm_start(timer *t, btime after)
+tm_set_in(timer *t, btime when, struct birdloop *loop)
 {
-  tm_set(t, current_time() + MAX(after, 0));
+  ASSERT_DIE(birdloop_inside(loop));
+  tm_set_in_tl(t, when, birdloop_time_loop(loop));
 }
 
 void
@@ -141,18 +134,23 @@ tm_stop(timer *t)
   if (!t->expires)
     return;
 
-  uint tc = timers_count(local_timeloop);
+  TLOCK_TIMER_ASSERT(t->loop);
 
-  HEAP_DELETE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
+  uint tc = timers_count(t->loop);
-  BUFFER_POP(local_timeloop->timers);
+
+  HEAP_DELETE(t->loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
+  BUFFER_POP(t->loop->timers);
 
   t->index = -1;
   t->expires = 0;
+  t->loop = NULL;
 }
 
 void
 timers_init(struct timeloop *loop, pool *p)
 {
+  TLOCK_TIMER_ASSERT(loop);
+
   BUFFER_INIT(loop->timers, p, 4);
   BUFFER_PUSH(loop->timers) = NULL;
 }
@@ -160,8 +158,10 @@ timers_init(struct timeloop *loop, pool *p)
 void io_log_event(void *hook, void *data);
 
 void
-timers_fire(struct timeloop *loop)
+timers_fire(struct timeloop *loop, int io_log)
 {
+  TLOCK_TIMER_ASSERT(loop);
+
   btime base_time;
   timer *t;
 
@@ -183,26 +183,19 @@ timers_fire(struct timeloop *loop)
       if (t->randomize)
 	when += random() % (t->randomize + 1);
 
-      tm_set(t, when);
+      tm_set_in_tl(t, when, loop);
     }
     else
       tm_stop(t);
 
     /* This is ugly hack, we want to log just timers executed from the main I/O loop */
-    if (loop == &main_timeloop)
+    if (io_log)
       io_log_event(t->hook, t->data);
 
     t->hook(t);
   }
 }
 
-void
-timer_init(void)
-{
-  timers_init(&main_timeloop, &root_pool);
-  local_timeloop = &main_timeloop;
-}
-
 
 /**
  * tm_parse_time - parse a date and time

lib/timer.h

@@ -12,6 +12,8 @@
 
 #include "nest/bird.h"
 #include "lib/buffer.h"
+#include "lib/io-loop.h"
+#include "lib/locking.h"
 #include "lib/resource.h"
 
 #include <stdatomic.h>
@@ -29,22 +31,27 @@ typedef struct timer
   uint randomize;			/* Amount of randomization */
   uint recurrent;			/* Timer recurrence */
 
+  struct timeloop *loop;		/* Loop where the timer is active */
+
   int index;
 } timer;
 
 struct timeloop
 {
   BUFFER_(timer *) timers;
+  struct domain_generic *domain;
+  struct birdloop *loop;
+  struct coroutine *coro;
 };
 
+#define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain))
+#define TLOCK_LOCAL_ASSERT(loop) ASSERT_DIE(!(loop)->domain || DG_IS_LOCKED((loop)->domain))
+
 static inline uint timers_count(struct timeloop *loop)
-{ return loop->timers.used - 1; }
+{ TLOCK_TIMER_ASSERT(loop); return loop->timers.used - 1; }
 
 static inline timer *timers_first(struct timeloop *loop)
-{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; }
+{ TLOCK_TIMER_ASSERT(loop); return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; }
-
-extern struct timeloop main_timeloop;
-extern _Thread_local struct timeloop *local_timeloop;
 
 #define current_time()		atomic_load_explicit(&last_time, memory_order_acquire)
 #define current_real_time()	atomic_load_explicit(&real_time, memory_order_acquire)
@@ -54,10 +61,13 @@ extern _Thread_local struct timeloop *local_timeloop;
 extern btime boot_time;
 
 timer *tm_new(pool *p);
-void tm_set(timer *t, btime when);
+#define tm_set(t, when) tm_set_in((t), (when), &main_birdloop)
-void tm_start(timer *t, btime after);
+#define tm_start(t, after) tm_start_in((t), (after), &main_birdloop)
 void tm_stop(timer *t);
 
+void tm_set_in(timer *t, btime when, struct birdloop *loop);
+#define tm_start_in(t, after, loop) tm_set_in((t), (current_time() + MAX_((after), 0)), loop)
+
 static inline int
 tm_active(timer *t)
 {
@@ -101,9 +111,7 @@ void times_update(void);
 
 /* For I/O loop */
 void timers_init(struct timeloop *loop, pool *p);
-void timers_fire(struct timeloop *loop);
+void timers_fire(struct timeloop *loop, int io_log);
-
-void timer_init(void);
 
 
 struct timeformat {

nest/a-path_test.c

@@ -204,10 +204,13 @@ t_as_path_converting(void)
 }
 #endif
 
+void resource_sys_init(void);
+
 int
 main(int argc, char *argv[])
 {
   bt_init(argc, argv);
+  resource_sys_init();
 
   bt_test_suite(t_as_path_match, "Testing AS path matching and some a-path utilities.");
   bt_test_suite(t_path_format, "Testing formating as path into byte buffer");

nest/a-set_test.c

@@ -240,10 +240,14 @@ t_set_ec_delete(void)
   return 1;
 }
 
+
+void resource_sys_init(void);
+
 int
 main(int argc, char *argv[])
 {
   bt_init(argc, argv);
+  resource_sys_init();
 
   bt_test_suite(t_set_int_contains, "Testing sets of integers: contains, get_data");
   bt_test_suite(t_set_int_format,   "Testing sets of integers: format");

nest/config.Y

@@ -434,6 +434,7 @@ proto: dev_proto '}' ;
 dev_proto_start: proto_start DIRECT {
      this_proto = proto_config_new(&proto_device, $1);
      init_list(&DIRECT_CFG->iface_list);
+     this_proto->late_if_feed = 1;
    }
  ;
 

nest/proto.c

@@ -15,6 +15,7 @@
 #include "lib/event.h"
 #include "lib/timer.h"
 #include "lib/string.h"
+#include "lib/coro.h"
 #include "conf/conf.h"
 #include "nest/route.h"
 #include "nest/iface.h"
@@ -58,7 +59,28 @@ static void channel_feed_end(struct channel *c);
 static void channel_export_stopped(struct rt_export_request *req);
 
 static inline int proto_is_done(struct proto *p)
-{ return (p->proto_state == PS_DOWN) && (p->active_channels == 0); }
+{ return (p->proto_state == PS_DOWN) && proto_is_inactive(p); }
+
+static inline event_list *proto_event_list(struct proto *p)
+{ return p->loop == &main_birdloop ? &global_event_list : birdloop_event_list(p->loop); }
+
+static inline event_list *proto_work_list(struct proto *p)
+{ return p->loop == &main_birdloop ? &global_work_list : birdloop_event_list(p->loop); }
+
+static inline void proto_send_event(struct proto *p)
+{ ev_send(proto_event_list(p), p->event); }
+
+#define PROTO_ENTER_FROM_MAIN(p)    ({ \
+    ASSERT_DIE(birdloop_inside(&main_birdloop)); \
+    struct birdloop *_loop = (p)->loop; \
+    if (_loop != &main_birdloop) birdloop_enter(_loop); \
+    _loop; \
+    })
+
+#define PROTO_LEAVE_FROM_MAIN(loop) ({ if (loop != &main_birdloop) birdloop_leave(loop); })
+
+#define PROTO_LOCKED_FROM_MAIN(p)	for (struct birdloop *_proto_loop = PROTO_ENTER_FROM_MAIN(p); _proto_loop; PROTO_LEAVE_FROM_MAIN(_proto_loop), (_proto_loop = NULL))
+
 
 static inline int channel_is_active(struct channel *c)
 { return (c->channel_state != CS_DOWN); }
@@ -473,6 +495,7 @@ channel_start_export(struct channel *c)
 
   c->out_req = (struct rt_export_request) {
     .name = rn,
+    .list = proto_work_list(c->proto),
     .trace_routes = c->debug | c->proto->debug,
     .dump_req = channel_dump_export_req,
     .log_state_change = channel_export_log_state_change,
@@ -517,7 +540,7 @@ channel_check_stopped(struct channel *c)
 	return;
 
       channel_set_state(c, CS_DOWN);
-      ev_schedule(c->proto->event);
+      proto_send_event(c->proto);
 
       break;
     case CS_PAUSE:
@@ -853,6 +876,7 @@ channel_setup_in_table(struct channel *c, int best)
   };
   c->in_table->get = (struct rt_export_request) {
     .name = cat->name,
+    .list = proto_work_list(c->proto),
     .trace_routes = c->debug | c->proto->debug,
     .dump_req = channel_in_get_dump_req,
     .log_state_change = channel_get_log_state_change,
@@ -895,6 +919,7 @@ channel_setup_out_table(struct channel *c)
   };
   c->out_table->get = (struct rt_export_request) {
     .name = cat->name,
+    .list = proto_work_list(c->proto),
     .trace_routes = c->debug | c->proto->debug,
     .dump_req = channel_out_get_dump_req,
     .log_state_change = channel_get_log_state_change,
@@ -997,7 +1022,7 @@ channel_do_down(struct channel *c)
 
   /* Schedule protocol shutddown */
   if (proto_is_done(c->proto))
-    ev_schedule(c->proto->event);
+    proto_send_event(c->proto);
 }
 
 void
@@ -1085,9 +1110,12 @@ channel_request_table_feeding(struct channel *c)
 void
 channel_request_feeding(struct channel *c)
 {
+  if (c->gr_wait || !c->proto->rt_notify)
+    return;
+
   CD(c, "Refeed requested");
 
-  ASSERT(c->out_req.hook);
+  ASSERT_DIE(c->out_req.hook);
 
   if (c->out_table)
     channel_aux_request_refeed(c->out_table);
@@ -1331,18 +1359,36 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con
   return 1;
 }
 
+static void
+proto_cleanup(struct proto *p)
+{
+  rfree(p->pool);
+  p->pool = NULL;
+
+  p->active = 0;
+  proto_log_state_change(p);
+  proto_rethink_goal(p);
+}
+
+static void
+proto_loop_stopped(void *ptr)
+{
+  struct proto *p = ptr;
+
+  birdloop_enter(&main_birdloop);
+
+  p->loop = &main_birdloop;
+  p->event->list = NULL;
+  proto_cleanup(p);
+
+  birdloop_leave(&main_birdloop);
+}
 
 static void
 proto_event(void *ptr)
 {
   struct proto *p = ptr;
 
-  if (p->do_start)
-  {
-    if_feed_baby(p);
-    p->do_start = 0;
-  }
-
   if (p->do_stop)
   {
     if (p->proto == &proto_unix_iface)
@@ -1351,14 +1397,10 @@ proto_event(void *ptr)
   }
 
   if (proto_is_done(p))
-  {
+    if (p->loop != &main_birdloop)
-    rfree(p->pool);
+      birdloop_stop_self(p->loop, proto_loop_stopped, p);
-    p->pool = NULL;
+    else
-
+      proto_cleanup(p);
-    p->active = 0;
-    proto_log_state_change(p);
-    proto_rethink_goal(p);
-  }
 }
 
 
@@ -1399,6 +1441,7 @@ proto_init(struct proto_config *c, node *n)
   struct protocol *pr = c->protocol;
   struct proto *p = pr->init(c);
 
+  p->loop = &main_birdloop;
   p->proto_state = PS_DOWN;
   p->last_state_change = current_time();
   p->vrf = c->vrf;
@@ -1415,11 +1458,27 @@ proto_init(struct proto_config *c, node *n)
 static void
 proto_start(struct proto *p)
 {
-  /* Here we cannot use p->cf->name since it won't survive reconfiguration */
+  DBG("Kicking %s up\n", p->name);
-  p->pool = rp_new(proto_pool, p->proto->name);
+  PD(p, "Starting");
+
+  int ns = strlen("Protocol ") + strlen(p->cf->name) + 1;
+  void *nb = mb_alloc(proto_pool, ns);
+  ASSERT_DIE(ns - 1 == bsnprintf(nb, ns, "Protocol %s", p->cf->name));
+
+  p->pool = rp_new(proto_pool, nb);
 
   if (graceful_restart_state == GRS_INIT)
     p->gr_recovery = 1;
+
+  if (p->cf->loop_order != DOMAIN_ORDER(the_bird))
+    p->loop = birdloop_new(p->pool, p->cf->loop_order, nb);
+
+  p->event->list = proto_event_list(p);
+
+  mb_move(nb, p->pool);
+
+  PROTO_LOCKED_FROM_MAIN(p)
+    proto_notify_state(p, (p->proto->start ? p->proto->start(p) : PS_UP));
 }
 
 
@@ -1455,6 +1514,7 @@ proto_config_new(struct protocol *pr, int class)
   cf->class = class;
   cf->debug = new_config->proto_default_debug;
   cf->mrtdump = new_config->proto_default_mrtdump;
+  cf->loop_order = DOMAIN_ORDER(the_bird);
 
   init_list(&cf->channels);
 
@@ -1743,12 +1803,21 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty
     proto_rethink_goal(p);
 }
 
+static void
+proto_shutdown(struct proto *p)
+{
+  if (p->proto_state == PS_START || p->proto_state == PS_UP)
+  {
+    /* Going down */
+    DBG("Kicking %s down\n", p->name);
+    PD(p, "Shutting down");
+    proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN));
+  }
+}
+
 static void
 proto_rethink_goal(struct proto *p)
 {
-  struct protocol *q;
-  byte goal;
-
   if (p->reconfiguring && !p->active)
   {
     struct proto_config *nc = p->cf_new;
@@ -1768,32 +1837,12 @@ proto_rethink_goal(struct proto *p)
 
   /* Determine what state we want to reach */
   if (p->disabled || p->reconfiguring)
-    goal = PS_DOWN;
-  else
-    goal = PS_UP;
-
-  q = p->proto;
-  if (goal == PS_UP)
   {
-    if (!p->active)
+    PROTO_LOCKED_FROM_MAIN(p)
-    {
+      proto_shutdown(p);
-      /* Going up */
+  }
-      DBG("Kicking %s up\n", p->name);
+  else if (!p->active)
-      PD(p, "Starting");
     proto_start(p);
-      proto_notify_state(p, (q->start ? q->start(p) : PS_UP));
-    }
-  }
-  else
-  {
-    if (p->proto_state == PS_START || p->proto_state == PS_UP)
-    {
-      /* Going down */
-      DBG("Kicking %s down\n", p->name);
-      PD(p, "Shutting down");
-      proto_notify_state(p, (q->shutdown ? q->shutdown(p) : PS_DOWN));
-    }
-  }
 }
 
 struct proto *
@@ -1998,7 +2047,7 @@ protos_dump_all(void)
 #define DPF(x)	(p->x ? " " #x : "")
     debug("  protocol %s (%p) state %s with %d active channels flags: %s%s%s%s\n",
 	p->name, p, p_states[p->proto_state], p->active_channels,
-	DPF(disabled), DPF(active), DPF(do_start), DPF(do_stop), DPF(reconfiguring));
+	DPF(disabled), DPF(active), DPF(do_stop), DPF(reconfiguring));
 #undef DPF
 
     struct channel *c;
@@ -2286,8 +2335,8 @@ static inline void
 proto_do_start(struct proto *p)
 {
   p->active = 1;
-  p->do_start = 1;
+  if (!p->cf->late_if_feed)
-  ev_schedule(p->event);
+    if_feed_baby(p);
 }
 
 static void
@@ -2300,6 +2349,9 @@ proto_do_up(struct proto *p)
   }
 
   proto_start_channels(p);
+
+  if (p->cf->late_if_feed)
+    if_feed_baby(p);
 }
 
 static inline void
@@ -2314,9 +2366,6 @@ proto_do_stop(struct proto *p)
   p->down_sched = 0;
   p->gr_recovery = 0;
 
-  p->do_stop = 1;
-  ev_schedule(p->event);
-
   if (p->main_source)
   {
     rt_unlock_source(p->main_source);
@@ -2324,6 +2373,9 @@ proto_do_stop(struct proto *p)
   }
 
   proto_stop_channels(p);
+
+  p->do_stop = 1;
+  proto_send_event(p);
 }
 
 static void
@@ -2334,7 +2386,7 @@ proto_do_down(struct proto *p)
 
   /* Shutdown is finished in the protocol event */
   if (proto_is_done(p))
-    ev_schedule(p->event);
+    proto_send_event(p);
 }
 
 
@@ -2573,7 +2625,7 @@ proto_cmd_disable(struct proto *p, uintptr_t arg, int cnt UNUSED)
   p->disabled = 1;
   p->down_code = PDC_CMD_DISABLE;
   proto_set_message(p, (char *) arg, -1);
-  proto_rethink_goal(p);
+  proto_shutdown(p);
   cli_msg(-9, "%s: disabled", p->name);
 }
 
@@ -2606,9 +2658,9 @@ proto_cmd_restart(struct proto *p, uintptr_t arg, int cnt UNUSED)
   p->disabled = 1;
   p->down_code = PDC_CMD_RESTART;
   proto_set_message(p, (char *) arg, -1);
-  proto_rethink_goal(p);
+  proto_shutdown(p);
   p->disabled = 0;
-  proto_rethink_goal(p);
+  /* After the protocol shuts down, proto_rethink_goal() is run from proto_event. */
   cli_msg(-12, "%s: restarted", p->name);
 }
 
@@ -2683,7 +2735,9 @@ proto_apply_cmd_symbol(const struct symbol *s, void (* cmd)(struct proto *, uint
 
   if (s->proto->proto)
   {
-    cmd(s->proto->proto, arg, 0);
+    struct proto *p = s->proto->proto;
+    PROTO_LOCKED_FROM_MAIN(p)
+      cmd(p, arg, 0);
     cli_msg(0, "");
   }
   else
@@ -2698,6 +2752,7 @@ proto_apply_cmd_patt(const char *patt, void (* cmd)(struct proto *, uintptr_t, i
 
   WALK_LIST(p, proto_list)
     if (!patt || patmatch(patt, p->name))
+      PROTO_LOCKED_FROM_MAIN(p)
 	cmd(p, arg, cnt++);
 
   if (!cnt)

nest/protocol.h

@@ -120,8 +120,10 @@ struct proto_config {
   u8 net_type;				/* Protocol network type (NET_*), 0 for undefined */
   u8 disabled;				/* Protocol enabled/disabled by default */
   u8 vrf_set;				/* Related VRF instance (below) is defined */
+  u8 late_if_feed;			/* Delay interface feed after channels are up */
   u32 debug, mrtdump;			/* Debugging bitfields, both use D_* constants */
   u32 router_id;			/* Protocol specific router ID */
+  uint loop_order;			/* Launch a birdloop on this locking level; use DOMAIN_ORDER(the_bird) for mainloop */
 
   list channels;			/* List of channel configs (struct channel_config) */
   struct iface *vrf;			/* Related VRF instance, NULL if global */
@@ -139,6 +141,7 @@ struct proto {
   struct proto_config *cf_new;		/* Configuration we want to switch to after shutdown (NULL=delete) */
   pool *pool;				/* Pool containing local objects */
   event *event;				/* Protocol event */
+  struct birdloop *loop;		/* BIRDloop running this protocol */
 
   list channels;			/* List of channels to rtables (struct channel) */
   struct channel *main_channel;		/* Primary channel */
@@ -149,12 +152,12 @@ struct proto {
   u32 debug;				/* Debugging flags */
   u32 mrtdump;				/* MRTDump flags */
   uint active_channels;			/* Number of active channels */
+  uint active_coroutines;		/* Number of active coroutines */
   byte net_type;			/* Protocol network type (NET_*), 0 for undefined */
   byte disabled;			/* Manually disabled */
   byte vrf_set;				/* Related VRF instance (above) is defined */
   byte proto_state;			/* Protocol state machine (PS_*, see below) */
   byte active;				/* From PS_START to cleanup after PS_STOP */
-  byte do_start;			/* Start actions are scheduled */
   byte do_stop;				/* Stop actions are scheduled */
   byte reconfiguring;			/* We're shutting down due to reconfiguration */
   byte gr_recovery;			/* Protocol should participate in graceful restart recovery */
@@ -356,6 +359,8 @@ void proto_notify_state(struct proto *p, unsigned state);
  *	as a result of received ROUTE-REFRESH request).
  */
 
+static inline int proto_is_inactive(struct proto *p)
+{ return (p->active_channels == 0) && (p->active_coroutines == 0); }
 
 
 /*

nest/route.h

@@ -11,6 +11,7 @@
 #define _BIRD_ROUTE_H_
 
 #include "lib/lists.h"
+#include "lib/event.h"
 #include "lib/bitmap.h"
 #include "lib/resource.h"
 #include "lib/net.h"
@@ -329,6 +330,8 @@ struct rt_export_request {
   char *name;
   u8 trace_routes;
 
+  event_list *list;			/* Where to schedule export events */
+
   /* There are two methods of export. You can either request feeding every single change
    * or feeding the whole route feed. In case of regular export, &export_one is preferred.
    * Anyway, when feeding, &export_bulk is preferred, falling back to &export_one.

nest/rt-table.c

@@ -1105,6 +1105,12 @@ rt_next_export(struct rt_export_hook *hook, rtable *tab)
     return tab->first_export;
 }
 
+static inline void
+rt_send_export_event(struct rt_export_hook *hook)
+{
+  ev_send(hook->req->list, hook->event);
+}
+
 static void
 rt_announce_exports(timer *tm)
 {
@@ -1116,7 +1122,7 @@ rt_announce_exports(timer *tm)
     if (atomic_load_explicit(&c->export_state, memory_order_acquire) != TES_READY)
       continue;
 
-    ev_schedule_work(c->event);
+    rt_send_export_event(c);
   }
 }
 
@@ -1169,7 +1175,7 @@ rt_export_hook(void *_data)
     rte_update_unlock();
   }
 
-  ev_schedule_work(c->event);
+  rt_send_export_event(c);
 }
 
 
@@ -1732,7 +1738,7 @@ rt_request_export(rtable *tab, struct rt_export_request *req)
   DBG("New export hook %p req %p in table %s uc=%u\n", hook, req, tab->name, tab->use_count);
 
   hook->event = ev_new_init(p, rt_feed_channel, hook);
-  ev_schedule_work(hook->event);
+  rt_send_export_event(hook);
 
   rt_set_export_state(hook, TES_FEEDING);
 }
@@ -1754,7 +1760,7 @@ rt_stop_export(struct rt_export_request *req, void (*stopped)(struct rt_export_r
   hook->event->hook = rt_export_stopped;
   hook->stopped = stopped;
 
-  ev_schedule(hook->event);
+  rt_send_export_event(hook);
 
   rt_set_export_state(hook, TES_STOP);
 }
@@ -2869,7 +2875,7 @@ rt_feed_channel(void *data)
       if (max_feed <= 0)
 	{
 	  FIB_ITERATE_PUT(fit);
-	  ev_schedule_work(c->event);
+	  rt_send_export_event(c);
 	  return;
 	}
 
@@ -2904,7 +2910,7 @@ rt_feed_channel(void *data)
   FIB_ITERATE_END;
 
   c->event->hook = rt_export_hook;
-  ev_schedule_work(c->event);
+  rt_send_export_event(c);
 
   rt_set_export_state(c, TES_READY);
 }

proto/bfd/bfd.c

@@ -113,8 +113,16 @@
 #define HASH_IP_EQ(a1,n1,a2,n2)	ipa_equal(a1, a2) && n1 == n2
 #define HASH_IP_FN(a,n)		ipa_hash(a) ^ u32_hash(n)
 
-static list bfd_proto_list;
+DEFINE_DOMAIN(rtable);
-static list bfd_wait_list;
+#define BFD_LOCK	LOCK_DOMAIN(rtable, bfd_global.lock)
+#define BFD_UNLOCK	UNLOCK_DOMAIN(rtable, bfd_global.lock)
+
+static struct {
+  DOMAIN(rtable) lock;
+  list wait_list;
+  list pickup_list;
+  list proto_list;
+} bfd_global;
 
 const char *bfd_state_names[] = { "AdminDown", "Down", "Init", "Up" };
 
@@ -188,7 +196,7 @@ bfd_session_update_tx_interval(struct bfd_session *s)
     return;
 
   /* Set timer relative to last tx_timer event */
-  tm_set(s->tx_timer, s->last_tx + tx_int_l);
+  tm_set_in(s->tx_timer, s->last_tx + tx_int_l, s->ifa->bfd->p.loop);
 }
 
 static void
@@ -202,7 +210,7 @@ bfd_session_update_detection_time(struct bfd_session *s, int kick)
   if (!s->last_rx)
     return;
 
-  tm_set(s->hold_timer, s->last_rx + timeout);
+  tm_set_in(s->hold_timer, s->last_rx + timeout, s->ifa->bfd->p.loop);
 }
 
 static void
@@ -226,7 +234,7 @@ bfd_session_control_tx_timer(struct bfd_session *s, int reset)
   if (reset || !tm_active(s->tx_timer))
   {
     s->last_tx = 0;
-    tm_start(s->tx_timer, 0);
+    tm_start_in(s->tx_timer, 0, s->ifa->bfd->p.loop);
   }
 
   return;
@@ -419,7 +427,7 @@ bfd_get_free_id(struct bfd_proto *p)
 static struct bfd_session *
 bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *iface, struct bfd_options *opts)
 {
-  birdloop_enter(p->loop);
+  ASSERT_DIE(birdloop_inside(p->p.loop));
 
   struct bfd_iface *ifa = bfd_get_iface(p, local, iface);
 
@@ -454,8 +462,6 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *
 
   TRACE(D_EVENTS, "Session to %I added", s->addr);
 
-  birdloop_leave(p->loop);
-
   return s;
 }
 
@@ -463,38 +469,34 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *
 static void
 bfd_open_session(struct bfd_proto *p, struct bfd_session *s, ip_addr local, struct iface *ifa)
 {
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
 
   s->opened = 1;
 
   bfd_session_control_tx_timer(s);
 
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 }
 
 static void
 bfd_close_session(struct bfd_proto *p, struct bfd_session *s)
 {
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
 
   s->opened = 0;
 
   bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_PATH_DOWN);
   bfd_session_control_tx_timer(s);
 
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 }
 */
 
 static void
-bfd_remove_session(struct bfd_proto *p, struct bfd_session *s)
+bfd_remove_session_locked(struct bfd_proto *p, struct bfd_session *s)
 {
-  ip_addr ip = s->addr;
-
   /* Caller should ensure that request list is empty */
 
-  birdloop_enter(p->loop);
-
   /* Remove session from notify list if scheduled for notification */
   /* No need for bfd_lock_sessions(), we are already protected by birdloop_enter() */
   if (NODE_VALID(&s->n))
@@ -508,11 +510,17 @@ bfd_remove_session(struct bfd_proto *p, struct bfd_session *s)
   HASH_REMOVE(p->session_hash_id, HASH_ID, s);
   HASH_REMOVE(p->session_hash_ip, HASH_IP, s);
 
+  TRACE(D_EVENTS, "Session to %I removed", s->addr);
+
   sl_free(p->session_slab, s);
+}
 
-  TRACE(D_EVENTS, "Session to %I removed", ip);
+static void
-
+bfd_remove_session(struct bfd_proto *p, struct bfd_session *s)
-  birdloop_leave(p->loop);
+{
+  birdloop_enter(p->p.loop);
+  bfd_remove_session_locked(p, s);
+  birdloop_leave(p->p.loop);
 }
 
 static void
@@ -521,7 +529,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s)
   if (EMPTY_LIST(s->request_list))
     return;
 
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
 
   struct bfd_request *req = SKIP_BACK(struct bfd_request, n, HEAD(s->request_list));
   s->cf = bfd_merge_options(s->ifa->cf, &req->opts);
@@ -534,7 +542,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s)
 
   bfd_session_control_tx_timer(s, 0);
 
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 
   TRACE(D_EVENTS, "Session to %I reconfigured", s->addr);
 }
@@ -618,9 +626,9 @@ bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_con
     (new->passive != old->passive);
 
   /* This should be probably changed to not access ifa->cf from the BFD thread */
-  birdloop_enter(p->loop);
+  birdloop_enter(p->p.loop);
   ifa->cf = new;
-  birdloop_leave(p->loop);
+  birdloop_leave(p->p.loop);
 }
 
 
@@ -681,41 +689,68 @@ bfd_add_request(struct bfd_proto *p, struct bfd_request *req)
 }
 
 static void
-bfd_submit_request(struct bfd_request *req)
+bfd_pickup_requests(void *_data UNUSED)
 {
   node *n;
+  WALK_LIST(n, bfd_global.proto_list)
+  {
+    struct bfd_proto *p = SKIP_BACK(struct bfd_proto, bfd_node, n);
+    birdloop_enter(p->p.loop);
+    BFD_LOCK;
 
-  WALK_LIST(n, bfd_proto_list)
+    node *rn, *rnxt;
-    if (bfd_add_request(SKIP_BACK(struct bfd_proto, bfd_node, n), req))
+    WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list)
-      return;
+      bfd_add_request(p, SKIP_BACK(struct bfd_request, n, rn));
 
-  rem_node(&req->n);
+    BFD_UNLOCK;
-  add_tail(&bfd_wait_list, &req->n);
+    birdloop_leave(p->p.loop);
-  req->session = NULL;
-  bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0);
   }
 
+  BFD_LOCK;
+  node *rn, *rnxt;
+  WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list)
+  {
+    rem_node(rn);
+    add_tail(&bfd_global.wait_list, rn);
+    bfd_request_notify(SKIP_BACK(struct bfd_request, n, rn), BFD_STATE_ADMIN_DOWN, 0);
+  }
+  BFD_UNLOCK;
+}
+
+static event bfd_pickup_event = { .hook = bfd_pickup_requests };
+
 static void
 bfd_take_requests(struct bfd_proto *p)
 {
   node *n, *nn;
-
+  BFD_LOCK;
-  WALK_LIST_DELSAFE(n, nn, bfd_wait_list)
+  WALK_LIST_DELSAFE(n, nn, bfd_global.wait_list)
     bfd_add_request(p, SKIP_BACK(struct bfd_request, n, n));
+  BFD_UNLOCK;
 }
 
 static void
 bfd_drop_requests(struct bfd_proto *p)
 {
   node *n;
-
+  BFD_LOCK;
-  HASH_WALK(p->session_hash_id, next_id, s)
+  HASH_WALK_DELSAFE(p->session_hash_id, next_id, s)
   {
-    /* We assume that p is not in bfd_proto_list */
     WALK_LIST_FIRST(n, s->request_list)
-      bfd_submit_request(SKIP_BACK(struct bfd_request, n, n));
+    {
+      struct bfd_request *req = SKIP_BACK(struct bfd_request, n, n);
+      rem_node(&req->n);
+      add_tail(&bfd_global.pickup_list, &req->n);
+      req->session = NULL;
+      bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0);
+    }
+
+    ev_send(&global_event_list, &bfd_pickup_event);
+
+    bfd_remove_session_locked(p, s);
   }
   HASH_WALK_END;
+  BFD_UNLOCK;
 }
 
 static struct resclass bfd_request_class;
@@ -728,9 +763,6 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local,
 {
   struct bfd_request *req = ralloc(p, &bfd_request_class);
 
-  /* Hack: self-link req->n, we will call rem_node() on it */
-  req->n.prev = req->n.next = &req->n;
-
   req->addr = addr;
   req->local = local;
   req->iface = iface;
@@ -739,11 +771,16 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local,
   if (opts)
     req->opts = *opts;
 
-  bfd_submit_request(req);
-
   req->hook = hook;
   req->data = data;
 
+  req->session = NULL;
+
+  BFD_LOCK;
+  add_tail(&bfd_global.pickup_list, &req->n);
+  ev_send(&global_event_list, &bfd_pickup_event);
+  BFD_UNLOCK;
+
   return req;
 }
 
@@ -1001,8 +1038,10 @@ bfd_notify_init(struct bfd_proto *p)
 void
 bfd_init_all(void)
 {
-  init_list(&bfd_proto_list);
+  bfd_global.lock = DOMAIN_NEW(rtable, "BFD Global");
-  init_list(&bfd_wait_list);
+  init_list(&bfd_global.wait_list);
+  init_list(&bfd_global.pickup_list);
+  init_list(&bfd_global.proto_list);
 }
 
 static struct proto *
@@ -1021,10 +1060,10 @@ bfd_start(struct proto *P)
   struct bfd_proto *p = (struct bfd_proto *) P;
   struct bfd_config *cf = (struct bfd_config *) (P->cf);
 
-  p->loop = birdloop_new();
-  p->tpool = rp_new(NULL, "BFD thread root");
   pthread_spin_init(&p->lock, PTHREAD_PROCESS_PRIVATE);
 
+  p->tpool = rp_new(P->pool, "BFD loop pool");
+
   p->session_slab = sl_new(P->pool, sizeof(struct bfd_session));
   HASH_INIT(p->session_hash_id, P->pool, 8);
   HASH_INIT(p->session_hash_ip, P->pool, 8);
@@ -1034,9 +1073,7 @@ bfd_start(struct proto *P)
   init_list(&p->notify_list);
   bfd_notify_init(p);
 
-  add_tail(&bfd_proto_list, &p->bfd_node);
+  add_tail(&bfd_global.proto_list, &p->bfd_node);
-
-  birdloop_enter(p->loop);
 
   if (cf->accept_ipv4 && cf->accept_direct)
     p->rx4_1 = bfd_open_rx_sk(p, 0, SK_IPV4);
@@ -1050,42 +1087,33 @@ bfd_start(struct proto *P)
   if (cf->accept_ipv6 && cf->accept_multihop)
     p->rx6_m = bfd_open_rx_sk(p, 1, SK_IPV6);
 
-  birdloop_leave(p->loop);
-
   bfd_take_requests(p);
 
   struct bfd_neighbor *n;
   WALK_LIST(n, cf->neigh_list)
     bfd_start_neighbor(p, n);
 
-  birdloop_start(p->loop);
-
   return PS_UP;
 }
 
-
 static int
 bfd_shutdown(struct proto *P)
 {
   struct bfd_proto *p = (struct bfd_proto *) P;
-  struct bfd_config *cf = (struct bfd_config *) (P->cf);
+  struct bfd_config *cf = (struct bfd_config *) (p->p.cf);
 
   rem_node(&p->bfd_node);
 
-  birdloop_stop(p->loop);
+  struct bfd_neighbor *bn;
-
+  WALK_LIST(bn, cf->neigh_list)
-  struct bfd_neighbor *n;
+    bfd_stop_neighbor(p, bn);
-  WALK_LIST(n, cf->neigh_list)
-    bfd_stop_neighbor(p, n);
 
   bfd_drop_requests(p);
 
-  /* FIXME: This is hack */
+  if (p->rx4_1) sk_stop(p->rx4_1);
-  birdloop_enter(p->loop);
+  if (p->rx4_m) sk_stop(p->rx4_m);
-  rfree(p->tpool);
+  if (p->rx6_1) sk_stop(p->rx6_1);
-  birdloop_leave(p->loop);
+  if (p->rx6_m) sk_stop(p->rx6_m);
-
-  birdloop_free(p->loop);
 
   return PS_DOWN;
 }
@@ -1105,7 +1133,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c)
       (new->accept_multihop != old->accept_multihop))
     return 0;
 
-  birdloop_mask_wakeups(p->loop);
+  birdloop_mask_wakeups(p->p.loop);
 
   WALK_LIST(ifa, p->iface_list)
     bfd_reconfigure_iface(p, ifa, new);
@@ -1119,7 +1147,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c)
 
   bfd_reconfigure_neighbors(p, new);
 
-  birdloop_unmask_wakeups(p->loop);
+  birdloop_unmask_wakeups(p->p.loop);
 
   return 1;
 }

proto/bfd/bfd.h

@@ -17,12 +17,12 @@
 #include "nest/password.h"
 #include "conf/conf.h"
 #include "lib/hash.h"
+#include "lib/io-loop.h"
 #include "lib/resource.h"
 #include "lib/socket.h"
 #include "lib/string.h"
 
 #include "nest/bfd.h"
-#include "sysdep/unix/io-loop.h"
 
 
 #define BFD_CONTROL_PORT	3784
@@ -87,9 +87,11 @@ struct bfd_neighbor
 struct bfd_proto
 {
   struct proto p;
-  struct birdloop *loop;
+
-  pool *tpool;
   pthread_spinlock_t lock;
+
+  pool *tpool;
+
   node bfd_node;
 
   slab *session_slab;

proto/bfd/config.Y

@@ -36,6 +36,7 @@ proto: bfd_proto ;
 bfd_proto_start: proto_start BFD
 {
   this_proto = proto_config_new(&proto_bfd, $1);
+  this_proto->loop_order = DOMAIN_ORDER(proto);
   init_list(&BFD_CFG->patt_list);
   init_list(&BFD_CFG->neigh_list);
   BFD_CFG->accept_ipv4 = BFD_CFG->accept_ipv6 = 1;

proto/bfd/packets.c

@@ -410,7 +410,7 @@ bfd_err_hook(sock *sk, int err)
 sock *
 bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af)
 {
-  sock *sk = sk_new(p->tpool);
+  sock *sk = sk_new(p->p.pool);
   sk->type = SK_UDP;
   sk->subtype = af;
   sk->sport = !multihop ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT;
@@ -441,7 +441,7 @@ bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af)
 sock *
 bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa)
 {
-  sock *sk = sk_new(p->tpool);
+  sock *sk = sk_new(p->p.pool);
   sk->type = SK_UDP;
   sk->saddr = local;
   sk->dport = ifa ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT;

proto/bgp/bgp.c

@@ -853,6 +853,7 @@ bgp_graceful_restart_feed(struct bgp_channel *c)
 {
   c->stale_feed = (struct rt_export_request) {
     .name = "BGP-GR",
+    .list = &global_work_list,
     .trace_routes = c->c.debug | c->c.proto->debug,
     .dump_req = bgp_graceful_restart_feed_dump_req,
     .log_state_change = bgp_graceful_restart_feed_log_state_change,

sysdep/unix/coroutine.c

@@ -21,10 +21,9 @@
 #include "lib/resource.h"
 #include "lib/timer.h"
 
-/* Using a rather big stack for coroutines to allow for stack-local allocations.
+#include "conf/conf.h"
- * In real world, the kernel doesn't alloc this memory until it is used.
+
- * */
+#define CORO_STACK_SIZE	65536
-#define CORO_STACK_SIZE	1048576
 
 /*
  *	Implementation of coroutines based on POSIX threads
@@ -79,6 +78,11 @@ domain_free(struct domain_generic *dg)
   xfree(dg);
 }
 
+uint dg_order(struct domain_generic *dg)
+{
+  return dg->order;
+}
+
 void do_lock(struct domain_generic *dg, struct domain_generic **lsp)
 {
   if ((char *) lsp - (char *) &locking_stack != dg->order)
@@ -89,7 +93,11 @@ void do_lock(struct domain_generic *dg, struct domain_generic **lsp)
   if (*lsp)
     bug("Inconsistent locking stack state on lock");
 
+  btime lock_begin = current_time();
   pthread_mutex_lock(&dg->mutex);
+  btime duration = current_time() - lock_begin;
+  if (config && (duration > config->watchdog_warning))
+    log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS));
 
   if (dg->prev || dg->locked_by)
     bug("Previous unlock not finished correctly");
@@ -140,11 +148,16 @@ static struct resclass coro_class = {
   .free = coro_free,
 };
 
+_Thread_local struct coroutine *this_coro = NULL;
+
 static void *coro_entry(void *p)
 {
   struct coroutine *c = p;
+
   ASSERT_DIE(c->entry);
 
+  this_coro = c;
+
   c->entry(c->data);
   ASSERT_DIE(coro_cleaned_up);
 

sysdep/unix/io-loop.c

@@ -15,50 +15,47 @@
 #include <sys/time.h>
 
 #include "nest/bird.h"
-#include "sysdep/unix/io-loop.h"
 
 #include "lib/buffer.h"
+#include "lib/coro.h"
 #include "lib/lists.h"
 #include "lib/resource.h"
 #include "lib/event.h"
 #include "lib/timer.h"
 #include "lib/socket.h"
 
-
+#include "lib/io-loop.h"
-struct birdloop
+#include "sysdep/unix/io-loop.h"
-{
+#include "conf/conf.h"
-  pool *pool;
-  pthread_t thread;
-  pthread_mutex_t mutex;
-
-  u8 stop_called;
-  u8 poll_active;
-  u8 wakeup_masked;
-  int wakeup_fds[2];
-
-  struct timeloop time;
-  list event_list;
-  list sock_list;
-  uint sock_num;
-
-  BUFFER(sock *) poll_sk;
-  BUFFER(struct pollfd) poll_fd;
-  u8 poll_changed;
-  u8 close_scheduled;
-};
-
 
 /*
  *	Current thread context
  */
 
-static _Thread_local struct birdloop *birdloop_current;
+_Thread_local struct birdloop *birdloop_current;
+static _Thread_local struct birdloop *birdloop_wakeup_masked;
+static _Thread_local uint birdloop_wakeup_masked_count;
 
-static inline void
+event_list *
-birdloop_set_current(struct birdloop *loop)
+birdloop_event_list(struct birdloop *loop)
 {
-  birdloop_current = loop;
+  return &loop->event_list;
-  local_timeloop = loop ? &loop->time : &main_timeloop;
+}
+
+struct timeloop *
+birdloop_time_loop(struct birdloop *loop)
+{
+  return &loop->time;
+}
+
+_Bool
+birdloop_inside(struct birdloop *loop)
+{
+  for (struct birdloop *c = birdloop_current; c; c = c->prev_loop)
+    if (loop == c)
+      return 1;
+
+  return 0;
 }
 
 /*
@@ -135,57 +132,17 @@ wakeup_do_kick(struct birdloop *loop)
   pipe_kick(loop->wakeup_fds[1]);
 }
 
-static inline void
+void
-wakeup_kick(struct birdloop *loop)
+birdloop_ping(struct birdloop *loop)
 {
-  if (!loop->wakeup_masked)
+  u32 ping_sent = atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel);
-    wakeup_do_kick(loop);
+  if (ping_sent)
+    return;
+
+  if (loop == birdloop_wakeup_masked)
+    birdloop_wakeup_masked_count++;
   else
-    loop->wakeup_masked = 2;
+    wakeup_do_kick(loop);
-}
-
-/* For notifications from outside */
-void
-wakeup_kick_current(void)
-{
-  if (birdloop_current && birdloop_current->poll_active)
-    wakeup_kick(birdloop_current);
-}
-
-
-/*
- *	Events
- */
-
-static inline uint
-events_waiting(struct birdloop *loop)
-{
-  return !EMPTY_LIST(loop->event_list);
-}
-
-static inline void
-events_init(struct birdloop *loop)
-{
-  init_list(&loop->event_list);
-}
-
-static void
-events_fire(struct birdloop *loop)
-{
-  times_update();
-  ev_run_list(&loop->event_list);
-}
-
-void
-ev2_schedule(event *e)
-{
-  if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list))
-    wakeup_kick(birdloop_current);
-
-  if (e->n.next)
-    rem_node(&e->n);
-
-  add_tail(&birdloop_current->event_list, &e->n);
 }
 
 
@@ -213,13 +170,13 @@ sockets_add(struct birdloop *loop, sock *s)
   s->index = -1;
   loop->poll_changed = 1;
 
-  if (loop->poll_active)
+  birdloop_ping(loop);
-    wakeup_kick(loop);
 }
 
 void
 sk_start(sock *s)
 {
+  ASSERT_DIE(birdloop_current != &main_birdloop);
   sockets_add(birdloop_current, s);
 }
 
@@ -230,28 +187,21 @@ sockets_remove(struct birdloop *loop, sock *s)
   loop->sock_num--;
 
   if (s->index >= 0)
+  {
     loop->poll_sk.data[s->index] = NULL;
-
     s->index = -1;
     loop->poll_changed = 1;
-
+    loop->close_scheduled = 1;
-  /* Wakeup moved to sk_stop() */
+    birdloop_ping(loop);
+  }
+  else
+    close(s->fd);
 }
 
 void
 sk_stop(sock *s)
 {
   sockets_remove(birdloop_current, s);
-
-  if (birdloop_current->poll_active)
-  {
-    birdloop_current->close_scheduled = 1;
-    wakeup_kick(birdloop_current);
-  }
-  else
-    close(s->fd);
-
-  s->fd = -1;
 }
 
 static inline uint sk_want_events(sock *s)
@@ -351,118 +301,185 @@ sockets_fire(struct birdloop *loop)
 
     if (pfd->revents & POLLIN)
       while (e && *psk && (*psk)->rx_hook)
-	e = sk_read(*psk, 0);
+	e = sk_read(*psk, pfd->revents);
 
     e = 1;
     if (pfd->revents & POLLOUT)
+    {
+      loop->poll_changed = 1;
       while (e && *psk)
 	e = sk_write(*psk);
     }
   }
+}
 
 
 /*
  *	Birdloop
  */
 
-static void *birdloop_main(void *arg);
+struct birdloop main_birdloop;
+
+static void birdloop_enter_locked(struct birdloop *loop);
+
+void
+birdloop_init(void)
+{
+  wakeup_init(&main_birdloop);
+
+  main_birdloop.time.domain = the_bird_domain.the_bird;
+  main_birdloop.time.loop = &main_birdloop;
+
+  times_update();
+  timers_init(&main_birdloop.time, &root_pool);
+
+  birdloop_enter_locked(&main_birdloop);
+}
+
+static void birdloop_main(void *arg);
 
 struct birdloop *
-birdloop_new(void)
+birdloop_new(pool *pp, uint order, const char *name)
 {
-  pool *p = rp_new(NULL, "Birdloop root");
+  struct domain_generic *dg = domain_new(name, order);
+
+  pool *p = rp_new(pp, name);
   struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop));
   loop->pool = p;
-  pthread_mutex_init(&loop->mutex, NULL);
+
+  loop->time.domain = dg;
+  loop->time.loop = loop;
+
+  birdloop_enter(loop);
 
   wakeup_init(loop);
-
+  ev_init_list(&loop->event_list, loop, name);
-  events_init(loop);
   timers_init(&loop->time, p);
   sockets_init(loop);
 
+  loop->time.coro = coro_run(p, birdloop_main, loop);
+
+  birdloop_leave(loop);
+
   return loop;
 }
 
-void
+static void
-birdloop_start(struct birdloop *loop)
+birdloop_do_stop(struct birdloop *loop, void (*stopped)(void *data), void *data)
 {
-  int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop);
+  loop->stopped = stopped;
-  if (rv)
+  loop->stop_data = data;
-    die("pthread_create(): %M", rv);
+  wakeup_do_kick(loop);
 }
 
 void
-birdloop_stop(struct birdloop *loop)
+birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data)
 {
-  pthread_mutex_lock(&loop->mutex);
+  DG_LOCK(loop->time.domain);
-  loop->stop_called = 1;
+  birdloop_do_stop(loop, stopped, data);
-  wakeup_do_kick(loop);
+  DG_UNLOCK(loop->time.domain);
-  pthread_mutex_unlock(&loop->mutex);
+}
 
-  int rv = pthread_join(loop->thread, NULL);
+void
-  if (rv)
+birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data)
-    die("pthread_join(): %M", rv);
+{
+  ASSERT_DIE(loop == birdloop_current);
+  ASSERT_DIE(DG_IS_LOCKED(loop->time.domain));
+
+  birdloop_do_stop(loop, stopped, data);
 }
 
 void
 birdloop_free(struct birdloop *loop)
 {
+  ASSERT_DIE(loop->links == 0);
+  domain_free(loop->time.domain);
   rfree(loop->pool);
 }
 
+static void
+birdloop_enter_locked(struct birdloop *loop)
+{
+  ASSERT_DIE(DG_IS_LOCKED(loop->time.domain));
+  ASSERT_DIE(!birdloop_inside(loop));
+
+  /* Store the old context */
+  loop->prev_loop = birdloop_current;
+
+  /* Put the new context */
+  birdloop_current = loop;
+}
 
 void
 birdloop_enter(struct birdloop *loop)
 {
-  /* TODO: these functions could save and restore old context */
+  DG_LOCK(loop->time.domain);
-  pthread_mutex_lock(&loop->mutex);
+  return birdloop_enter_locked(loop);
-  birdloop_set_current(loop);
+}
+
+static void
+birdloop_leave_locked(struct birdloop *loop)
+{
+  /* Check the current context */
+  ASSERT_DIE(birdloop_current == loop);
+
+  /* Restore the old context */
+  birdloop_current = loop->prev_loop;
 }
 
 void
 birdloop_leave(struct birdloop *loop)
 {
-  /* TODO: these functions could save and restore old context */
+  birdloop_leave_locked(loop);
-  birdloop_set_current(NULL);
+  DG_UNLOCK(loop->time.domain);
-  pthread_mutex_unlock(&loop->mutex);
 }
 
 void
 birdloop_mask_wakeups(struct birdloop *loop)
 {
-  pthread_mutex_lock(&loop->mutex);
+  ASSERT_DIE(birdloop_wakeup_masked == NULL);
-  loop->wakeup_masked = 1;
+  birdloop_wakeup_masked = loop;
-  pthread_mutex_unlock(&loop->mutex);
 }
 
 void
 birdloop_unmask_wakeups(struct birdloop *loop)
 {
-  pthread_mutex_lock(&loop->mutex);
+  ASSERT_DIE(birdloop_wakeup_masked == loop);
-  if (loop->wakeup_masked == 2)
+  birdloop_wakeup_masked = NULL;
+  if (birdloop_wakeup_masked_count)
     wakeup_do_kick(loop);
-  loop->wakeup_masked = 0;
+
-  pthread_mutex_unlock(&loop->mutex);
+  birdloop_wakeup_masked_count = 0;
 }
 
-static void *
+void
+birdloop_link(struct birdloop *loop)
+{
+  ASSERT_DIE(birdloop_inside(loop));
+  loop->links++;
+}
+
+void
+birdloop_unlink(struct birdloop *loop)
+{
+  ASSERT_DIE(birdloop_inside(loop));
+  loop->links--;
+}
+
+static void
 birdloop_main(void *arg)
 {
   struct birdloop *loop = arg;
   timer *t;
   int rv, timeout;
 
-  birdloop_set_current(loop);
+  btime loop_begin = current_time();
 
-  pthread_mutex_lock(&loop->mutex);
+  birdloop_enter(loop);
   while (1)
   {
-    events_fire(loop);
+    timers_fire(&loop->time, 0);
-    timers_fire(&loop->time);
+    if (ev_run_list(&loop->event_list))
-
-    times_update();
-    if (events_waiting(loop))
       timeout = 0;
     else if (t = timers_first(&loop->time))
       timeout = (tm_remains(t) TO_MS) + 1;
@@ -472,8 +489,11 @@ birdloop_main(void *arg)
     if (loop->poll_changed)
       sockets_prepare(loop);
 
-    loop->poll_active = 1;
+    btime duration = current_time() - loop_begin;
-    pthread_mutex_unlock(&loop->mutex);
+    if (duration > config->watchdog_warning)
+      log(L_WARN "I/O loop cycle took %d ms", (int) (duration TO_MS));
+
+    birdloop_leave(loop);
 
   try:
     rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout);
@@ -484,25 +504,32 @@ birdloop_main(void *arg)
       die("poll: %m");
     }
 
-    pthread_mutex_lock(&loop->mutex);
+    birdloop_enter(loop);
-    loop->poll_active = 0;
 
     if (loop->close_scheduled)
       sockets_close_fds(loop);
 
-    if (loop->stop_called)
+    if (loop->stopped)
       break;
 
+    loop_begin = current_time();
+
     if (rv)
       sockets_fire(loop);
 
-    timers_fire(&loop->time);
+    atomic_exchange_explicit(&loop->ping_sent, 0, memory_order_acq_rel);
   }
 
-  loop->stop_called = 0;
+  /* Flush remaining events */
-  pthread_mutex_unlock(&loop->mutex);
+  ASSERT_DIE(!ev_run_list(&loop->event_list));
 
-  return NULL;
+  /* No timers allowed */
+  ASSERT_DIE(timers_count(&loop->time) == 0);
+  ASSERT_DIE(EMPTY_LIST(loop->sock_list));
+  ASSERT_DIE(loop->sock_num == 0);
+
+  birdloop_leave(loop);
+  loop->stopped(loop->stop_data);
 }
 
 

sysdep/unix/io-loop.h

@@ -4,31 +4,32 @@
  *	Can be freely distributed and used under the terms of the GNU GPL.
  */
 
-#ifndef _BIRD_IO_LOOP_H_
+#ifndef _BIRD_SYSDEP_UNIX_IO_LOOP_H_
-#define _BIRD_IO_LOOP_H_
+#define _BIRD_SYSDEP_UNIX_IO_LOOP_H_
 
-#include "nest/bird.h"
+struct birdloop
-#include "lib/lists.h"
+{
-#include "lib/resource.h"
+  pool *pool;
-#include "lib/event.h"
-#include "lib/timer.h"
-#include "lib/socket.h"
 
+  struct timeloop time;
+  event_list event_list;
+  list sock_list;
+  uint sock_num;
 
-void ev2_schedule(event *e);
+  BUFFER(sock *) poll_sk;
+  BUFFER(struct pollfd) poll_fd;
+  u8 poll_changed;
+  u8 close_scheduled;
 
-void sk_start(sock *s);
+  _Atomic u32 ping_sent;
-void sk_stop(sock *s);
+  int wakeup_fds[2];
 
-struct birdloop *birdloop_new(void);
+  uint links;
-void birdloop_start(struct birdloop *loop);
-void birdloop_stop(struct birdloop *loop);
-void birdloop_free(struct birdloop *loop);
 
-void birdloop_enter(struct birdloop *loop);
+  void (*stopped)(void *data);
-void birdloop_leave(struct birdloop *loop);
+  void *stop_data;
-void birdloop_mask_wakeups(struct birdloop *loop);
-void birdloop_unmask_wakeups(struct birdloop *loop);
 
+  struct birdloop *prev_loop;
+};
 
-#endif /* _BIRD_IO_LOOP_H_ */
+#endif

sysdep/unix/io.c

@@ -43,6 +43,7 @@
 #include "conf/conf.h"
 
 #include "sysdep/unix/unix.h"
+#include "sysdep/unix/io-loop.h"
 #include CONFIG_INCLUDE_SYSIO_H
 
 /* Maximum number of calls of tx handler for one socket in one
@@ -800,18 +801,16 @@ sk_free(resource *r)
     sk_ssh_free(s);
 #endif
 
-  if (s->fd < 0)
+  if ((s->fd < 0) || (s->flags & SKF_THREAD))
     return;
 
-  /* FIXME: we should call sk_stop() for SKF_THREAD sockets */
-  if (!(s->flags & SKF_THREAD))
-  {
   if (s == current_sock)
     current_sock = sk_next(s);
   if (s == stored_sock)
     stored_sock = sk_next(s);
+
+  if (enlisted(&s->n))
     rem_node(&s->n);
-  }
 
   if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE)
     close(s->fd);
@@ -1104,7 +1103,11 @@ sk_passive_connected(sock *s, int type)
     return 1;
   }
 
+  if (s->flags & SKF_PASSIVE_THREAD)
+    t->flags |= SKF_THREAD;
+  else
     sk_insert(t);
+
   sk_alloc_bufs(t);
   s->rx_hook(t, 0);
   return 1;
@@ -1508,6 +1511,36 @@ sk_open_unix(sock *s, char *name)
   return 0;
 }
 
+static void
+sk_reloop_hook(void *_vs)
+{
+  sock *s = _vs;
+  if (birdloop_inside(&main_birdloop))
+  {
+    s->flags &= ~SKF_THREAD;
+    sk_insert(s);
+  }
+  else
+  {
+    s->flags |= SKF_THREAD;
+    sk_start(s);
+  }
+}
+
+void
+sk_reloop(sock *s, struct birdloop *loop)
+{
+  if (enlisted(&s->n))
+    rem_node(&s->n);
+
+  s->reloop = (event) {
+    .hook = sk_reloop_hook,
+    .data = s,
+  };
+
+  ev_send_loop(loop, &s->reloop);
+}
+
 
 #define CMSG_RX_SPACE MAX(CMSG4_SPACE_PKTINFO+CMSG4_SPACE_TTL, \
 			  CMSG6_SPACE_PKTINFO+CMSG6_SPACE_TTL)
@@ -2143,8 +2176,9 @@ void
 io_init(void)
 {
   init_list(&sock_list);
-  init_list(&global_event_list);
+  ev_init_list(&global_event_list, &main_birdloop, "Global event list");
-  init_list(&global_work_list);
+  ev_init_list(&global_work_list, &main_birdloop, "Global work list");
+  ev_init_list(&main_birdloop.event_list, &main_birdloop, "Global fast event list");
   krt_io_init();
   // XXX init_times();
   // XXX update_times();
@@ -2158,14 +2192,7 @@ static int short_loops = 0;
 #define SHORT_LOOP_MAX 10
 #define WORK_EVENTS_MAX 10
 
-static int poll_reload_pipe[2];
+void pipe_drain(int fd);
-
-void
-io_loop_reload(void)
-{
-  char b;
-  write(poll_reload_pipe[1], &b, 1);
-}
 
 void
 io_loop(void)
@@ -2178,21 +2205,19 @@ io_loop(void)
   int fdmax = 256;
   struct pollfd *pfd = xmalloc(fdmax * sizeof(struct pollfd));
 
-  if (pipe(poll_reload_pipe) < 0)
-    die("pipe(poll_reload_pipe) failed: %m");
-
   watchdog_start1();
   for(;;)
     {
       times_update();
       events = ev_run_list(&global_event_list);
       events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events;
-      timers_fire(&main_timeloop);
+      events = ev_run_list(&main_birdloop.event_list) || events;
+      timers_fire(&main_birdloop.time, 1);
       io_close_event();
 
       // FIXME
       poll_tout = (events ? 0 : 3000); /* Time in milliseconds */
-      if (t = timers_first(&main_timeloop))
+      if (t = timers_first(&main_birdloop.time))
       {
 	times_update();
 	timeout = (tm_remains(t) TO_MS) + 1;
@@ -2200,7 +2225,7 @@ io_loop(void)
       }
 
       /* A hack to reload main io_loop() when something has changed asynchronously. */
-      pfd[0].fd = poll_reload_pipe[0];
+      pfd[0].fd = main_birdloop.wakeup_fds[0];
       pfd[0].events = POLLIN;
 
       nfds = 1;
@@ -2263,9 +2288,9 @@ io_loop(void)
 
       /* And finally enter poll() to find active sockets */
       watchdog_stop();
-      the_bird_unlock();
+      birdloop_leave(&main_birdloop);
       pout = poll(pfd, nfds, poll_tout);
-      the_bird_lock();
+      birdloop_enter(&main_birdloop);
       watchdog_start();
 
       if (pout < 0)
@@ -2279,8 +2304,8 @@ io_loop(void)
 	  if (pfd[0].revents & POLLIN)
 	  {
 	    /* IO loop reload requested */
-	    char b;
+	    pipe_drain(main_birdloop.wakeup_fds[0]);
-	    read(poll_reload_pipe[0], &b, 1);
+	    atomic_exchange_explicit(&main_birdloop.ping_sent, 0, memory_order_acq_rel);
 	    continue;
 	  }
 

sysdep/unix/main.c

@@ -908,10 +908,12 @@ main(int argc, char **argv)
   parse_args(argc, argv);
   log_switch(1, NULL, NULL);
 
+  the_bird_lock();
+
   random_init();
   net_init();
   resource_init();
-  timer_init();
+  birdloop_init();
   olock_init();
   io_init();
   rt_init();
@@ -961,7 +963,6 @@ main(int argc, char **argv)
       dup2(0, 2);
     }
 
-  the_bird_lock();
 
   main_thread_init();
 

sysdep/unix/unix.h

@@ -106,7 +106,6 @@ extern volatile sig_atomic_t async_shutdown_flag;
 
 void io_init(void);
 void io_loop(void);
-void io_loop_reload(void);
 void io_log_dump(void);
 int sk_open_unix(struct birdsock *s, char *name);
 struct rfile *rf_open(struct pool *, const char *name, const char *mode);

test/bt-utils.c

@@ -53,16 +53,20 @@ cf_file_read(byte *dest, uint max_len, int fd)
   return l;
 }
 
+void resource_sys_init(void);
+
 void
 bt_bird_init(void)
 {
+  resource_sys_init();
   if(bt_verbose)
     log_init_debug("");
   log_switch(bt_verbose != 0, NULL, NULL);
 
+  the_bird_lock();
   resource_init();
   olock_init();
-  timer_init();
+  birdloop_init();
   io_init();
   rt_init();
   if_init();
@@ -79,6 +83,7 @@ void bt_bird_cleanup(void)
     class_to_protocol[i] = NULL;
 
   config = new_config = NULL;
+  the_bird_unlock();
 }
 
 static char *