Channel: Refeeding by an auxiliary request if needed.

If the protocol supports route refresh on export, we keep the stop-start
method of route refeed. This applies for BGP with ERR or with export
table on, for OSPF, Babel, RIP or Pipe.

For BGP without ERR or for future selective ROA reloads, we're adding an
auxiliary export request, doing the refeed while the main export request
is running, somehow resembling the original method of BIRD 2 refeed.

There is also a refeed request queue to keep track of different refeed
requests.

lib/bitmap.c

@@ -36,6 +36,8 @@ bmap_reset(struct bmap *b, uint size)
 void
 bmap_grow(struct bmap *b, uint need)
 {
+  ASSERT_DIE(b->size);
+
   uint size = b->size * 2;
   while (size < need)
     size *= 2;

nest/proto.c

@@ -54,6 +54,7 @@ static void channel_reset_limit(struct channel *c, struct limit *l, int dir);
 static void channel_feed_end(struct channel *c);
 static void channel_stop_export(struct channel *c);
 static void channel_export_stopped(struct rt_export_request *req);
+static void channel_refeed_stopped(struct rt_export_request *req);
 static void channel_check_stopped(struct channel *c);
 
 static inline int proto_is_done(struct proto *p)
@@ -88,7 +89,7 @@ channel_export_log_state_change(struct rt_export_request *req, u8 state)
   {
     case TES_FEEDING:
       if (c->proto->feed_begin)
-	c->proto->feed_begin(c, !c->refeeding);
+	c->proto->feed_begin(c);
       break;
     case TES_READY:
       channel_feed_end(c);
@@ -96,6 +97,25 @@ channel_export_log_state_change(struct rt_export_request *req, u8 state)
   }
 }
 
+void
+channel_refeed_log_state_change(struct rt_export_request *req, u8 state)
+{
+  struct channel *c = SKIP_BACK(struct channel, refeed_req, req);
+  CD(c, "Channel export state changed to %s", rt_export_state_name(state));
+
+  switch (state)
+  {
+    case TES_FEEDING:
+      if (c->proto->feed_begin)
+	c->proto->feed_begin(c);
+      break;
+    case TES_READY:
+      rt_stop_export(req, channel_refeed_stopped);
+      break;
+  }
+}
+
+
 static void
 channel_dump_import_req(struct rt_import_request *req)
 {
@@ -110,6 +130,40 @@ channel_dump_export_req(struct rt_export_request *req)
   debug("  Channel %s.%s export request %p\n", c->proto->name, c->name, req);
 }
 
+static void
+channel_dump_refeed_req(struct rt_export_request *req)
+{
+  struct channel *c = SKIP_BACK(struct channel, refeed_req, req);
+  debug("  Channel %s.%s refeed request %p\n", c->proto->name, c->name, req);
+}
+
+
+static void
+channel_rpe_mark_seen_export(struct rt_export_request *req, struct rt_pending_export *rpe)
+{
+  channel_rpe_mark_seen(SKIP_BACK(struct channel, out_req, req), rpe);
+}
+
+static void
+channel_rpe_mark_seen_refeed(struct rt_export_request *req, struct rt_pending_export *rpe)
+{
+  channel_rpe_mark_seen(SKIP_BACK(struct channel, refeed_req, req), rpe);
+}
+
+
+struct channel *
+channel_from_export_request(struct rt_export_request *req)
+{
+  if (req->dump_req == channel_dump_export_req)
+    return SKIP_BACK(struct channel, out_req, req);
+
+  if (req->dump_req == channel_dump_refeed_req)
+    return SKIP_BACK(struct channel, refeed_req, req);
+
+  bug("Garbled channel export request");
+}
+
+
 static void
 proto_log_state_change(struct proto *p)
 {
@@ -289,6 +343,7 @@ struct roa_subscription {
   struct settle settle;
   struct channel *c;
   struct rt_export_request req;
+  struct channel_feeding_request cfr[2];
 };
 
 static void
@@ -309,8 +364,23 @@ channel_roa_out_changed(struct settle *se)
 
   CD(c, "Feeding triggered by RPKI change");
 
-  c->refeed_pending = 1;
+  /* Refeed already pending */
-  channel_stop_export(c);
+  if ((s->cfr[0].state == CFRS_PENDING) || (s->cfr[1].state == CFRS_PENDING))
+    return;
+
+  /* First refeed inactive */
+  if (s->cfr[0].state == CFRS_INACTIVE)
+  {
+    s->cfr[0].type = CFRT_AUXILIARY;
+    channel_request_feeding(c, &s->cfr[0]);
+  }
+  else
+  {
+    /* Second refeed MUST be inactive */
+    ASSERT_DIE(s->cfr[1].state == CFRS_INACTIVE);
+    s->cfr[1].type = CFRT_AUXILIARY;
+    channel_request_feeding(c, &s->cfr[1]);
+  }
 }
 
 static void
@@ -505,7 +575,7 @@ channel_start_export(struct channel *c)
     .trace_routes = c->debug | c->proto->debug,
     .dump_req = channel_dump_export_req,
     .log_state_change = channel_export_log_state_change,
-    .mark_seen = channel_rpe_mark_seen,
+    .mark_seen = channel_rpe_mark_seen_export,
   };
 
   bmap_init(&c->export_map, c->proto->pool, 16);
@@ -533,6 +603,12 @@ channel_start_export(struct channel *c)
       bug("Unknown route announcement mode");
   }
 
+  c->refeed_req = c->out_req;
+  c->refeed_req.name = mb_sprintf(c->proto->pool, "%s.%s.refeed", c->proto->name, c->name);
+  c->refeed_req.dump_req = channel_dump_refeed_req;
+  c->refeed_req.log_state_change = channel_refeed_log_state_change;
+  c->refeed_req.mark_seen = channel_rpe_mark_seen_refeed;
+
   DBG("%s.%s: Channel start export req=%p\n", c->proto->name, c->name, &c->out_req);
   rt_request_export(c->table, &c->out_req);
 }
@@ -543,7 +619,7 @@ channel_check_stopped(struct channel *c)
   switch (c->channel_state)
   {
     case CS_STOP:
-      if (!EMPTY_LIST(c->roa_subscriptions) || c->out_req.hook || c->in_req.hook || c->reload_req.hook)
+      if (!EMPTY_LIST(c->roa_subscriptions) || c->out_req.hook || c->refeed_req.hook || c->in_req.hook || c->reload_req.hook)
 	return;
 
       channel_set_state(c, CS_DOWN);
@@ -551,7 +627,7 @@ channel_check_stopped(struct channel *c)
 
       break;
     case CS_PAUSE:
-      if (!EMPTY_LIST(c->roa_subscriptions) || c->out_req.hook || c->reload_req.hook)
+      if (!EMPTY_LIST(c->roa_subscriptions) || c->out_req.hook || c->refeed_req.hook || c->reload_req.hook)
 	return;
 
       channel_set_state(c, CS_START);
@@ -582,8 +658,9 @@ channel_export_stopped(struct rt_export_request *req)
 
   if (c->refeed_pending)
   {
-    c->refeeding = 1;
+    ASSERT_DIE(!c->refeeding);
-    c->refeed_pending = 0;
+    c->refeeding = c->refeed_pending;
+    c->refeed_pending = NULL;
 
     channel_reset_limit(c, &c->out_limit, PLD_OUT);
 
@@ -603,6 +680,34 @@ channel_export_stopped(struct rt_export_request *req)
   channel_check_stopped(c);
 }
 
+static void
+channel_refeed_stopped(struct rt_export_request *req)
+{
+  struct channel *c = SKIP_BACK(struct channel, refeed_req, req);
+
+  req->hook = NULL;
+
+  channel_feed_end(c);
+}
+
+static void
+channel_init_feeding(struct channel *c)
+{
+  for (struct channel_feeding_request *cfrp = c->refeed_pending; cfrp; cfrp = cfrp->next)
+    if (cfrp->type == CFRT_DIRECT)
+    {
+      /* Direct feeding requested? Restart the export by force. */
+      channel_stop_export(c);
+      return;
+    }
+
+  /* No direct feeding, running auxiliary refeed. */
+  c->refeeding = c->refeed_pending;
+  c->refeed_pending = NULL;
+  c->refeed_trie = f_new_trie(lp_new(c->proto->pool), 0);
+  rt_request_export(c->table, &c->refeed_req);
+}
+
 static void
 channel_feed_end(struct channel *c)
 {
@@ -610,24 +715,46 @@ channel_feed_end(struct channel *c)
   struct limit *l = &c->out_limit;
   if (c->refeeding &&
       (c->limit_active & (1 << PLD_OUT)) &&
-      (c->refeed_count <= l->max) &&
       (l->count <= l->max))
   {
     log(L_INFO "Protocol %s resets route export limit (%u)", c->proto->name, l->max);
-    channel_reset_limit(c, &c->out_limit, PLD_OUT);
+    c->limit_active &= ~(1 << PLD_OUT);
 
-    c->refeed_pending = 1;
+    /* Queue the same refeed batch back into pending */
-    channel_stop_export(c);
+    struct channel_feeding_request **ptr = &c->refeed_pending;
-    return;
+    while (*ptr)
+      ptr = &((*ptr)->next);
+
+    *ptr = c->refeeding;
+
+    /* Mark the requests to be redone */
+    for (struct channel_feeding_request *cfr = c->refeeding; cfr; cfr = cfr->next)
+      cfr->state = CFRS_PENDING;
+
+    c->refeeding = NULL;
   }
 
-  if (c->proto->feed_end)
+  /* Inform the protocol about the feed ending */
-    c->proto->feed_end(c);
+  CALL(c->proto->feed_end, c);
 
+  /* Free the dynamic feeding requests */
+  for (struct channel_feeding_request *cfr = c->refeeding, *next = cfr ? cfr->next : NULL;
+      cfr;
+      (cfr = next), (next = next ? next->next : NULL))
+    if (cfr->flags & CFRF_DYNAMIC)
+      mb_free(cfr);
+
+  /* Drop the refeed batch */
+  c->refeeding = NULL;
+  if (c->refeed_trie)
+  {
+    rfree(c->refeed_trie->lp);
+    c->refeed_trie = NULL;
+  }
+
+  /* Run the pending batch */
   if (c->refeed_pending)
-    channel_stop_export(c);
+    channel_init_feeding(c);
-  else
-    c->refeeding = 0;
 }
 
 /* Called by protocol for reload from in_table */
@@ -852,24 +979,36 @@ channel_set_state(struct channel *c, uint state)
  * even when feeding is already running, in that case it is restarted.
  */
 void
-channel_request_feeding(struct channel *c)
+channel_request_feeding(struct channel *c, struct channel_feeding_request *cfr)
 {
   ASSERT(c->out_req.hook);
 
-  if (c->refeed_pending)
+  /* Enqueue the request */
-    return;
+  cfr->next = c->refeed_pending;
+  c->refeed_pending = cfr;
 
-  c->refeed_pending = 1;
+  /* Initialize refeeds unless already refeeding */
-  channel_stop_export(c);
+  if (!c->refeeding)
+    channel_init_feeding(c);
+}
+
+void
+channel_request_feeding_dynamic(struct channel *c, enum channel_feeding_request_type type)
+{
+  struct channel_feeding_request *req = mb_allocz(c->proto->pool, sizeof *req);
+  req->type = type;
+  req->flags |= CFRF_DYNAMIC;
+  channel_request_feeding(c, req);
 }
 
 static void
 channel_stop_export(struct channel *c)
 {
-  if (!c->out_req.hook || (c->out_req.hook->export_state == TES_STOP))
+  if (c->refeed_req.hook && (c->refeed_req.hook->export_state != TES_STOP))
-    return;
+    rt_stop_export(&c->refeed_req, channel_refeed_stopped);
 
-  rt_stop_export(&c->out_req, channel_export_stopped);
+  if (c->out_req.hook && (c->out_req.hook->export_state != TES_STOP))
+    rt_stop_export(&c->out_req, channel_export_stopped);
 }
 
 static void
@@ -1073,7 +1212,7 @@ channel_reconfigure(struct channel *c, struct channel_config *cf)
     channel_request_reload(c);
 
   if (export_changed)
-    channel_request_feeding(c);
+    channel_request_feeding_dynamic(c, CFRT_AUXILIARY);
 
 done:
   CD(c, "Reconfigured");
@@ -2511,7 +2650,7 @@ proto_cmd_reload(struct proto *p, uintptr_t dir, int cnt UNUSED)
   if (dir != CMD_RELOAD_IN)
     WALK_LIST(c, p->channels)
       if (c->channel_state == CS_UP)
-	channel_request_feeding(c);
+	channel_request_feeding_dynamic(c, CFRT_AUXILIARY);
 
   cli_msg(-15, "%s: reloading", p->name);
 }

nest/protocol.h

@@ -17,6 +17,7 @@
 #include "nest/rt.h"
 #include "nest/limit.h"
 #include "conf/conf.h"
+#include "filter/data.h"
 
 struct iface;
 struct ifa;
@@ -194,7 +195,7 @@ struct proto {
   void (*rt_notify)(struct proto *, struct channel *, const net_addr *net, struct rte *new, const struct rte *old);
   int (*preexport)(struct channel *, struct rte *rt);
   void (*reload_routes)(struct channel *);
-  void (*feed_begin)(struct channel *, int initial);
+  void (*feed_begin)(struct channel *);
   void (*feed_end)(struct channel *);
 
   /*
@@ -568,7 +569,10 @@ struct channel {
   struct rt_import_request in_req;	/* Table import connection */
   struct rt_export_request out_req;	/* Table export connection */
 
-  u32 refeed_count;			/* Number of routes exported during refeed regardless of out_limit */
+  struct rt_export_request refeed_req;	/* Auxiliary refeed request */
+  struct f_trie *refeed_trie;		/* Auxiliary refeed trie */
+  struct channel_feeding_request *refeeding;	/* Refeeding the channel */
+  struct channel_feeding_request *refeed_pending;	/* Scheduled refeeds */
 
   uint feed_block_size;			/* How many routes to feed at once */
 
@@ -582,7 +586,6 @@ struct channel {
   u8 stale;				/* Used in reconfiguration */
 
   u8 channel_state;
-  u8 refeeding;				/* Refeeding the channel. */
   u8 reloadable;			/* Hook reload_routes() is allowed on the channel */
   u8 gr_lock;				/* Graceful restart mechanism should wait for this channel */
   u8 gr_wait;				/* Route export to channel is postponed until graceful restart */
@@ -592,7 +595,6 @@ struct channel {
   struct rt_export_request reload_req;	/* Feeder for import reload */
 
   u8 reload_pending;			/* Reloading and another reload is scheduled */
-  u8 refeed_pending;			/* Refeeding and another refeed is scheduled */
   u8 rpki_reload;			/* RPKI changes trigger channel reload */
 
   struct rt_exporter *out_table;	/* Internal table for exported routes */
@@ -673,7 +675,33 @@ static inline void channel_init(struct channel *c) { channel_set_state(c, CS_STA
 static inline void channel_open(struct channel *c) { channel_set_state(c, CS_UP); }
 static inline void channel_close(struct channel *c) { channel_set_state(c, CS_STOP); }
 
-void channel_request_feeding(struct channel *c);
+struct channel_feeding_request {
+  struct channel_feeding_request *next;
+  PACKED enum channel_feeding_request_type {
+    CFRT_DIRECT = 1,
+    CFRT_AUXILIARY,
+  } type;
+  PACKED enum {
+    CFRS_INACTIVE = 0,
+    CFRS_PENDING,
+    CFRS_RUNNING,
+  } state;
+  PACKED enum {
+    CFRF_DYNAMIC = 1,
+  } flags;
+};
+
+struct channel *channel_from_export_request(struct rt_export_request *req);
+void channel_request_feeding(struct channel *c, struct channel_feeding_request *);
+void channel_request_feeding_dynamic(struct channel *c, enum channel_feeding_request_type);
+
+static inline int channel_net_is_refeeding(struct channel *c, const net_addr *n)
+{ return (c->refeeding && c->refeed_trie && !trie_match_net(c->refeed_trie, n)); }
+static inline void channel_net_mark_refed(struct channel *c, const net_addr *n)
+{
+  ASSERT_DIE(c->refeeding && c->refeed_trie);
+  trie_add_prefix(c->refeed_trie, n, n->pxlen, n->pxlen);
+}
 void *channel_config_new(const struct channel_class *cc, const char *name, uint net_type, struct proto_config *proto);
 void *channel_config_get(const struct channel_class *cc, const char *name, uint net_type, struct proto_config *proto);
 int channel_reconfigure(struct channel *c, struct channel_config *cf);

nest/rt-show.c

@@ -142,7 +142,7 @@ rt_show_net(struct rt_show_data *d, const net_addr *n, const rte **feed, uint co
 	{
 	  /* Special case for merged export */
 	  pass = 1;
-	  rte *em = rt_export_merged(ec, feed, count, tmp_linpool, 1);
+	  rte *em = rt_export_merged(ec, n, feed, count, tmp_linpool, 1);
 
 	  if (em)
 	    e = *em;

nest/rt-table.c

@@ -858,9 +858,6 @@ do_rt_notify(struct channel *c, const net_addr *net, rte *new, const rte *old)
   struct proto *p = c->proto;
   struct channel_export_stats *stats = &c->export_stats;
 
-  if (c->refeeding && new)
-    c->refeed_count++;
-
   if (!old && new)
     if (CHANNEL_LIMIT_PUSH(c, OUT))
     {
@@ -897,9 +894,9 @@ do_rt_notify(struct channel *c, const net_addr *net, rte *new, const rte *old)
 }
 
 static void
-rt_notify_basic(struct channel *c, const net_addr *net, rte *new, const rte *old)
+rt_notify_basic(struct channel *c, const net_addr *net, rte *new, const rte *old, int force)
 {
-  if (new && old && rte_same(new, old))
+  if (new && old && rte_same(new, old) && !force)
   {
     channel_rte_trace_out(D_ROUTES, c, new, "already exported");
 
@@ -911,6 +908,10 @@ rt_notify_basic(struct channel *c, const net_addr *net, rte *new, const rte *old
     return;
   }
 
+  /* Refeeding and old is new */
+  if (force && !old && bmap_test(&c->export_map, new->id))
+    old = new;
+
   if (new)
     new = export_filter(c, new, 0);
 
@@ -924,13 +925,16 @@ rt_notify_basic(struct channel *c, const net_addr *net, rte *new, const rte *old
 }
 
 void
-channel_rpe_mark_seen(struct rt_export_request *req, struct rt_pending_export *rpe)
+channel_rpe_mark_seen(struct channel *c, struct rt_pending_export *rpe)
 {
-  struct channel *c = SKIP_BACK(struct channel, out_req, req);
-
   channel_trace(c, D_ROUTES, "Marking seen %p (%lu)", rpe, rpe->seq);
 
-  rpe_mark_seen(req->hook, rpe);
+  ASSERT_DIE(c->out_req.hook);
+  rpe_mark_seen(c->out_req.hook, rpe);
+
+  if (c->refeed_req.hook && (c->refeed_req.hook->export_state == TES_FEEDING))
+    rpe_mark_seen(c->refeed_req.hook, rpe);
+
   if (rpe->old)
     bmap_clear(&c->export_reject_map, rpe->old->rte.id);
 }
@@ -940,7 +944,8 @@ rt_notify_accepted(struct rt_export_request *req, const net_addr *n,
     struct rt_pending_export *first, struct rt_pending_export *last,
     const rte **feed, uint count)
 {
-  struct channel *c = SKIP_BACK(struct channel, out_req, req);
+  struct channel *c = channel_from_export_request(req);
+  int refeeding = channel_net_is_refeeding(c, n);
 
   rte nb0, *new_best = NULL;
   const rte *old_best = NULL;
@@ -951,22 +956,27 @@ rt_notify_accepted(struct rt_export_request *req, const net_addr *n,
       continue;
 
     /* Has been already rejected, won't bother with it */
-    if (!c->refeeding && bmap_test(&c->export_reject_map, feed[i]->id))
+    if (!refeeding && bmap_test(&c->export_reject_map, feed[i]->id))
       continue;
 
     /* Previously exported */
     if (!old_best && bmap_test(&c->export_map, feed[i]->id))
     {
-      /* is still best */
+      if (new_best)
-      if (!new_best)
       {
+	/* is superseded */
+	old_best = feed[i];
+	break;
+      }
+      else if (refeeding)
+	/* is superseeded but maybe by a new version of itself */
+	old_best = feed[i];
+      else
+      {
+	/* is still best */
 	DBG("rt_notify_accepted: idempotent\n");
 	goto done;
       }
-
-      /* is superseded */
-      old_best = feed[i];
-      break;
     }
 
     /* Have no new best route yet */
@@ -983,7 +993,7 @@ done:
   /* Check obsolete routes for previously exported */
   RPE_WALK(first, rpe, NULL)
   {
-    channel_rpe_mark_seen(req, rpe);
+    channel_rpe_mark_seen(c, rpe);
     if (rpe->old)
     {
       if (bmap_test(&c->export_map, rpe->old->rte.id))
@@ -1001,13 +1011,21 @@ done:
     do_rt_notify(c, n, new_best, old_best);
   else
     DBG("rt_notify_accepted: nothing to export\n");
+
+  if (refeeding)
+    channel_net_mark_refed(c, n);
 }
 
 rte *
-rt_export_merged(struct channel *c, const rte **feed, uint count, linpool *pool, int silent)
+rt_export_merged(struct channel *c, const net_addr *n, const rte **feed, uint count, linpool *pool, int silent)
 {
   _Thread_local static rte rloc;
 
+  int refeeding = !silent && channel_net_is_refeeding(c, n);
+
+  if (refeeding)
+    channel_net_mark_refed(c, n);
+
   // struct proto *p = c->proto;
   struct nexthop_adata *nhs = NULL;
   const rte *best0 = feed[0];
@@ -1017,7 +1035,7 @@ rt_export_merged(struct channel *c, const rte **feed, uint count, linpool *pool,
     return NULL;
 
   /* Already rejected, no need to re-run the filter */
-  if (!c->refeeding && bmap_test(&c->export_reject_map, best0->id))
+  if (!refeeding && bmap_test(&c->export_reject_map, best0->id))
     return NULL;
 
   rloc = *best0;
@@ -1037,7 +1055,7 @@ rt_export_merged(struct channel *c, const rte **feed, uint count, linpool *pool,
       continue;
 
     rte tmp0 = *feed[i];
-    rte *tmp = export_filter(c, &tmp0, 1);
+    rte *tmp = export_filter(c, &tmp0, !refeeding);
 
     if (!tmp || !rte_is_reachable(tmp))
       continue;
@@ -1070,7 +1088,7 @@ rt_notify_merged(struct rt_export_request *req, const net_addr *n,
     struct rt_pending_export *first, struct rt_pending_export *last,
     const rte **feed, uint count)
 {
-  struct channel *c = SKIP_BACK(struct channel, out_req, req);
+  struct channel *c = channel_from_export_request(req);
 
   // struct proto *p = c->proto;
 
@@ -1095,7 +1113,7 @@ rt_notify_merged(struct rt_export_request *req, const net_addr *n,
   /* Check obsolete routes for previously exported */
   RPE_WALK(first, rpe, NULL)
   {
-    channel_rpe_mark_seen(req, rpe);
+    channel_rpe_mark_seen(c, rpe);
     if (rpe->old)
     {
       if (bmap_test(&c->export_map, rpe->old->rte.id))
@@ -1109,7 +1127,7 @@ rt_notify_merged(struct rt_export_request *req, const net_addr *n,
   }
 
   /* Prepare new merged route */
-  rte *new_merged = count ? rt_export_merged(c, feed, count, tmp_linpool, 0) : NULL;
+  rte *new_merged = count ? rt_export_merged(c, n, feed, count, tmp_linpool, 0) : NULL;
 
   if (new_merged || old_best)
     do_rt_notify(c, n, new_merged, old_best);
@@ -1118,25 +1136,30 @@ rt_notify_merged(struct rt_export_request *req, const net_addr *n,
 void
 rt_notify_optimal(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *first)
 {
-  struct channel *c = SKIP_BACK(struct channel, out_req, req);
+  struct channel *c = channel_from_export_request(req);
   const rte *o = RTE_VALID_OR_NULL(first->old_best);
   struct rte_storage *new_best = first->new_best;
 
+  int refeeding = channel_net_is_refeeding(c, net);
+
   RPE_WALK(first, rpe, NULL)
   {
-    channel_rpe_mark_seen(req, rpe);
+    channel_rpe_mark_seen(c, rpe);
     new_best = rpe->new_best;
   }
 
   rte n0 = RTE_COPY_VALID(new_best);
   if (n0.src || o)
-    rt_notify_basic(c, net, n0.src ? &n0 : NULL, o);
+    rt_notify_basic(c, net, n0.src ? &n0 : NULL, o, refeeding);
+
+  if (refeeding)
+    channel_net_mark_refed(c, net);
 }
 
 void
 rt_notify_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *first)
 {
-  struct channel *c = SKIP_BACK(struct channel, out_req, req);
+  struct channel *c = channel_from_export_request(req);
 
   const rte *n = RTE_VALID_OR_NULL(first->new);
   const rte *o = RTE_VALID_OR_NULL(first->old);
@@ -1145,9 +1168,13 @@ rt_notify_any(struct rt_export_request *req, const net_addr *net, struct rt_pend
       "Notifying any, net %N, first %p (%lu), new %p, old %p",
       net, first, first->seq, n, o);
 
-  if (!n && !o)
+  if (!n && !o || channel_net_is_refeeding(c, net))
   {
-    channel_rpe_mark_seen(req, first);
+    /* We want to skip this notification because:
+     * - there is nothing to notify, or
+     * - this net is going to get a full refeed soon
+     */
+    channel_rpe_mark_seen(c, first);
     return;
   }
 
@@ -1156,13 +1183,13 @@ rt_notify_any(struct rt_export_request *req, const net_addr *net, struct rt_pend
 
   RPE_WALK(first, rpe, src)
   {
-    channel_rpe_mark_seen(req, rpe);
+    channel_rpe_mark_seen(c, rpe);
     new_latest = rpe->new;
   }
 
   rte n0 = RTE_COPY_VALID(new_latest);
   if (n0.src || o)
-    rt_notify_basic(c, net, n0.src ? &n0 : NULL, o);
+    rt_notify_basic(c, net, n0.src ? &n0 : NULL, o, 0);
 
   channel_trace(c, D_ROUTES, "Notified net %N", net);
 }
@@ -1172,7 +1199,8 @@ rt_feed_any(struct rt_export_request *req, const net_addr *net,
     struct rt_pending_export *first, struct rt_pending_export *last,
     const rte **feed, uint count)
 {
-  struct channel *c = SKIP_BACK(struct channel, out_req, req);
+  struct channel *c = channel_from_export_request(req);
+  int refeeding = channel_net_is_refeeding(c, net);
 
   channel_trace(c, D_ROUTES, "Feeding any, net %N, first %p (%lu), %p (%lu), count %u",
       net, first, first ? first->seq : 0, last ? last->seq : 0, count);
@@ -1181,17 +1209,19 @@ rt_feed_any(struct rt_export_request *req, const net_addr *net,
     if (rte_is_valid(feed[i]))
     {
       rte n0 = *feed[i];
-      rt_notify_basic(c, net, &n0, NULL);
+      rt_notify_basic(c, net, &n0, NULL, refeeding);
     }
 
   RPE_WALK(first, rpe, NULL)
   {
-    channel_rpe_mark_seen(req, rpe);
+    channel_rpe_mark_seen(c, rpe);
     if (rpe == last)
       break;
   }
 
   channel_trace(c, D_ROUTES, "Fed %N", net);
+  if (refeeding)
+    channel_net_mark_refed(c, net);
 }
 
 void

nest/rt.h

@@ -473,7 +473,7 @@ void rt_feed_any(struct rt_export_request *req, const net_addr *net, struct rt_p
 void rt_notify_accepted(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *first, struct rt_pending_export *last, const rte **feed, uint count);
 void rt_notify_merged(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *first, struct rt_pending_export *last, const rte **feed, uint count);
 
-void channel_rpe_mark_seen(struct rt_export_request *req, struct rt_pending_export *rpe);
+void channel_rpe_mark_seen(struct channel *c, struct rt_pending_export *rpe);
 
 /* Types of route announcement, also used as flags */
 #define RA_UNDEF	0		/* Undefined RA type */
@@ -598,7 +598,7 @@ static inline net *net_find_valid(struct rtable_private *tab, const net_addr *ad
 static inline net *net_get(struct rtable_private *tab, const net_addr *addr) { return (net *) fib_get(&tab->fib, addr); }
 net *net_route(struct rtable_private *tab, const net_addr *n);
 int rt_examine(rtable *t, net_addr *a, struct channel *c, const struct filter *filter);
-rte *rt_export_merged(struct channel *c, const rte ** feed, uint count, linpool *pool, int silent);
+rte *rt_export_merged(struct channel *c, const net_addr *n, const rte ** feed, uint count, linpool *pool, int silent);
 void rt_refresh_begin(struct rt_import_request *);
 void rt_refresh_end(struct rt_import_request *);
 void rt_modify_stale(rtable *t, struct rt_import_request *);

proto/babel/babel.c

@@ -2437,9 +2437,9 @@ babel_rt_notify(struct proto *P, struct channel *c UNUSED, const net_addr *net,
 }
 
 static void
-babel_feed_begin(struct channel *C, int initial)
+babel_feed_begin(struct channel *C)
 {
-  if (initial)
+  if (!C->refeeding || C->refeed_req.hook)
     return;
 
   struct babel_proto *p = (struct babel_proto *) C->proto;
@@ -2454,6 +2454,9 @@ babel_feed_begin(struct channel *C, int initial)
 static void
 babel_feed_end(struct channel *C)
 {
+  if (!C->refeeding || C->refeed_req.hook)
+    return;
+
   struct babel_proto *p = (struct babel_proto *) C->proto;
   struct fib *rtable = (C->net_type == NET_IP4) ? &p->ip4_rtable : &p->ip6_rtable;
   int changed = 0;

proto/bgp/bgp.c

@@ -1575,7 +1575,7 @@ bgp_reload_routes(struct channel *C)
 }
 
 static void
-bgp_feed_begin(struct channel *C, int initial)
+bgp_feed_begin(struct channel *C)
 {
   struct bgp_proto *p = (void *) C->proto;
   struct bgp_channel *c = (void *) C;
@@ -1588,21 +1588,28 @@ bgp_feed_begin(struct channel *C, int initial)
   if (!p->conn)
     return;
 
-  if (initial && p->cf->gr_mode)
+  if (!C->refeeding)
-    c->feed_state = BFS_LOADING;
-
-  if (!initial && C->out_table)
   {
-    c->feed_out_table = 1;
+    if (p->cf->gr_mode)
+      c->feed_state = BFS_LOADING;
     return;
   }
 
-  /* It is refeed and both sides support enhanced route refresh */
+  if (!C->refeed_req.hook)
-  if (!initial && p->enhanced_refresh)
   {
-    /* BoRR must not be sent before End-of-RIB */
+    /* Direct refeed */
-    if (c->feed_state == BFS_LOADING || c->feed_state == BFS_LOADED)
+    if (C->out_table)
+    {
+      /* FIXME: THIS IS BROKEN, IT DOESN'T PRUNE THE OUT TABLE */
+      c->feed_out_table = 1;
       return;
+    }
+
+    ASSERT_DIE(p->enhanced_refresh);
+
+    /* It is refeed and both sides support enhanced route refresh */
+    /* BoRR must not be sent before End-of-RIB */
+    ASSERT_DIE((c->feed_state != BFS_LOADING) && (c->feed_state != BFS_LOADED));
 
     c->feed_state = BFS_REFRESHING;
     bgp_schedule_packet(p->conn, c, PKT_BEGIN_REFRESH);

proto/bgp/packets.c

@@ -2801,7 +2801,12 @@ bgp_rx_route_refresh(struct bgp_conn *conn, byte *pkt, uint len)
   {
   case BGP_RR_REQUEST:
     BGP_TRACE(D_PACKETS, "Got ROUTE-REFRESH");
-    channel_request_feeding(&c->c);
+    if (c->c.out_table)
+    {
+      /* FIXME: REQUEST REFRESH FROM OUT TABLE */
+    }
+    else
+      channel_request_feeding_dynamic(&c->c, p->enhanced_refresh ? CFRT_DIRECT : CFRT_AUXILIARY);
     break;
 
   case BGP_RR_BEGIN:

proto/ospf/topology.c

@@ -551,8 +551,11 @@ ospf_update_lsadb(struct ospf_proto *p)
 }
 
 void
-ospf_feed_begin(struct channel *C, int initial UNUSED)
+ospf_feed_begin(struct channel *C)
 {
+  if (!C->refeeding || C->refeed_req.hook)
+    return;
+
   struct ospf_proto *p = (struct ospf_proto *) C->proto;
   struct top_hash_entry *en;
 
@@ -565,6 +568,9 @@ ospf_feed_begin(struct channel *C, int initial UNUSED)
 void
 ospf_feed_end(struct channel *C)
 {
+  if (!C->refeeding || C->refeed_req.hook)
+    return;
+
   struct ospf_proto *p = (struct ospf_proto *) C->proto;
   struct top_hash_entry *en;
 

proto/ospf/topology.h

@@ -187,7 +187,7 @@ struct top_hash_entry * ospf_originate_lsa(struct ospf_proto *p, struct ospf_new
 void ospf_advance_lsa(struct ospf_proto *p, struct top_hash_entry *en, struct ospf_lsa_header *lsa, u32 type, u32 domain, void *body);
 void ospf_flush_lsa(struct ospf_proto *p, struct top_hash_entry *en);
 void ospf_update_lsadb(struct ospf_proto *p);
-void ospf_feed_begin(struct channel *C, int initial);
+void ospf_feed_begin(struct channel *C);
 void ospf_feed_end(struct channel *C);
 
 static inline void ospf_flush2_lsa(struct ospf_proto *p, struct top_hash_entry **en)

proto/perf/perf.c

@@ -217,7 +217,7 @@ perf_rt_notify(struct proto *P, struct channel *c UNUSED, const net_addr *net UN
 }
 
 static void
-perf_feed_begin(struct channel *c, int initial UNUSED)
+perf_feed_begin(struct channel *c)
 {
   struct perf_proto *p = (struct perf_proto *) c->proto;
 
@@ -232,6 +232,7 @@ static void
 perf_feed_end(struct channel *c)
 {
   struct perf_proto *p = (struct perf_proto *) c->proto;
+
   struct timespec ts_end;
   clock_gettime(CLOCK_MONOTONIC, &ts_end);
 
@@ -243,7 +244,7 @@ perf_feed_end(struct channel *c)
   p->feed_begin = NULL;
 
   if (p->run < p->repeat)
-    channel_request_feeding(c);
+    channel_request_feeding_dynamic(c, CFRT_DIRECT);
   else
     PLOG("feed done");
 }

proto/pipe/pipe.c

@@ -102,12 +102,15 @@ pipe_reload_routes(struct channel *C)
   struct pipe_proto *p = (void *) C->proto;
 
   /* Route reload on one channel is just refeed on the other */
-  channel_request_feeding((C == p->pri) ? p->sec : p->pri);
+  channel_request_feeding_dynamic((C == p->pri) ? p->sec : p->pri, CFRT_DIRECT);
 }
 
 static void
-pipe_feed_begin(struct channel *C, int initial UNUSED)
+pipe_feed_begin(struct channel *C)
 {
+  if (!C->refeeding || C->refeed_req.hook)
+    return;
+
   struct pipe_proto *p = (void *) C->proto;
   uint *flags = (C == p->pri) ? &p->sec_flags : &p->pri_flags;
 
@@ -117,6 +120,9 @@ pipe_feed_begin(struct channel *C, int initial UNUSED)
 static void
 pipe_feed_end(struct channel *C)
 {
+  if (!C->refeeding || C->refeed_req.hook)
+    return;
+
   struct pipe_proto *p = (void *) C->proto;
   struct channel *dst = (C == p->pri) ? p->sec : p->pri;
   uint *flags = (C == p->pri) ? &p->sec_flags : &p->pri_flags;

proto/radv/radv.c

@@ -664,7 +664,7 @@ radv_reconfigure(struct proto *P, struct proto_config *CF)
 
   /* We started to accept routes so we need to refeed them */
   if (!old->propagate_routes && new->propagate_routes)
-    channel_request_feeding(p->p.main_channel);
+    channel_request_feeding_dynamic(p->p.main_channel, CFRT_DIRECT);
 
   IFACE_WALK(iface)
   {

proto/rip/rip.c

@@ -419,9 +419,9 @@ rip_rt_notify(struct proto *P, struct channel *ch UNUSED, const net_addr *net, s
 }
 
 void
-rip_feed_begin(struct channel *C, int initial)
+rip_feed_begin(struct channel *C)
 {
-  if (initial)
+  if (!C->refeeding || C->refeed_req.hook)
     return;
 
   struct rip_proto *p = (struct rip_proto *) C->proto;
@@ -437,6 +437,9 @@ rip_feed_begin(struct channel *C, int initial)
 void
 rip_feed_end(struct channel *C)
 {
+  if (!C->refeeding || C->refeed_req.hook)
+    return;
+
   struct rip_proto *p = (struct rip_proto *) C->proto;
   int changed = 0;
 

sysdep/unix/krt.c

@@ -389,7 +389,7 @@ krt_export_net(struct krt_proto *p, net *net)
 
     const rte **feed = alloca(count * sizeof(rte *));
     rte_feed_obtain_valid(net, feed, count);
-    return rt_export_merged(c, feed, count, krt_filter_lp, 1);
+    return rt_export_merged(c, net->n.addr, feed, count, krt_filter_lp, 1);
   }
 
   static _Thread_local rte rt;
@@ -787,6 +787,9 @@ krt_feed_end(struct channel *C)
 {
   struct krt_proto *p = (void *) C->proto;
 
+  if (C->refeeding && C->refeed_req.hook)
+    return;
+
   if (p->flush_routes)
   {
     p->flush_routes = 2;
@@ -922,7 +925,7 @@ krt_shutdown(struct proto *P)
   if (p->initialized && !KRT_CF->persist && (P->down_code != PDC_CMD_GR_DOWN))
   {
     p->flush_routes = 1;
-    channel_request_feeding(p->p.main_channel);
+    channel_request_feeding_dynamic(p->p.main_channel, CFRT_AUXILIARY);
     return PS_UP;
   }
   else