Protocols have now assigned startup phases

For now, there are 4 phases: Necessary (device), Connector (kernel, pipe), Generator (static, rpki) and Regular.
Started and reconfigured are from Necessary to Regular, shutdown backwards.

This way, kernel can flush routes before actually being shutdown.

nest/proto.c

@@ -1134,6 +1134,7 @@ proto_cleanup(struct proto *p)
 
   p->active = 0;
   proto_log_state_change(p);
+
   proto_rethink_goal(p);
 }
 
@@ -1422,6 +1423,18 @@ proto_reconfigure(struct proto *p, struct proto_config *oc, struct proto_config
   return 1;
 }
 
+static struct protos_commit_request {
+  struct config *new;
+  struct config *old;
+  enum protocol_startup phase;
+  int force_reconfig;
+  int type;
+} protos_commit_request;
+
+static int proto_rethink_goal_pending = 0;
+
+static void protos_do_commit(struct config *new, struct config *old, int force_reconfig, int type);
+
 /**
  * protos_commit - commit new protocol configuration
  * @new: new configuration
@@ -1453,15 +1466,39 @@ proto_reconfigure(struct proto *p, struct proto_config *oc, struct proto_config
 void
 protos_commit(struct config *new, struct config *old, int force_reconfig, int type)
 {
+  protos_commit_request = (struct protos_commit_request) {
+    .new = new,
+    .old = old,
+    .phase = (new->shutdown && !new->gr_down) ? PROTOCOL_STARTUP_REGULAR : PROTOCOL_STARTUP_NECESSARY,
+    .force_reconfig = force_reconfig,
+    .type = type,
+  };
+
+  protos_do_commit(new, old, force_reconfig, type);
+}
+
+static void
+protos_do_commit(struct config *new, struct config *old, int force_reconfig, int type)
+{
+  enum protocol_startup phase = protos_commit_request.phase;
   struct proto_config *oc, *nc;
   struct symbol *sym;
   struct proto *p;
 
+  if ((phase < PROTOCOL_STARTUP_REGULAR) || (phase > PROTOCOL_STARTUP_NECESSARY))
+  {
+    protos_commit_request = (struct protos_commit_request) {};
+    return;
+  }
+
   DBG("protos_commit:\n");
   if (old)
   {
     WALK_LIST(oc, old->protos)
     {
+      if (oc->protocol->startup != phase)
+	continue;
+
       p = oc->proto;
       sym = cf_find_symbol(new, oc->name);
 
@@ -1541,11 +1578,10 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty
     }
   }
 
-  struct proto *first_dev_proto = NULL;
   struct proto *after = NULL;
 
   WALK_LIST(nc, new->protos)
-    if (!nc->proto)
+    if ((nc->protocol->startup == phase) && !nc->proto)
     {
       /* Not a first-time configuration */
       if (old)
@@ -1554,18 +1590,13 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty
       p = proto_init(nc, after);
       after = p;
 
-      if (p->proto == &proto_unix_iface)
+      proto_rethink_goal(p);
-	first_dev_proto = p;
     }
     else
       after = nc->proto;
 
   DBG("Protocol start\n");
 
-  /* Start device protocol first */
-  if (first_dev_proto)
-    proto_rethink_goal(first_dev_proto);
-
   /* Determine router ID for the first time - it has to be here and not in
      global_commit() because it is postponed after start of device protocol */
   if (!config->router_id)
@@ -1575,9 +1606,15 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty
       die("Cannot determine router ID, please configure it manually");
   }
 
-  /* Start all new protocols */
+  /* Commit next round of protocols */
-  WALK_TLIST_DELSAFE(proto, p, &global_proto_list)
+  if (new->shutdown && !new->gr_down)
-    proto_rethink_goal(p);
+    protos_commit_request.phase++;
+  else
+    protos_commit_request.phase--;
+
+  /* If something is pending, the next round will be called asynchronously from proto_rethink_goal(). */
+  if (!proto_rethink_goal_pending)
+    protos_do_commit(new, old, force_reconfig, type);
 }
 
 static void
@@ -1589,12 +1626,19 @@ proto_shutdown(struct proto *p)
     DBG("Kicking %s down\n", p->name);
     PD(p, "Shutting down");
     proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN));
+    if (p->reconfiguring)
+    {
+      proto_rethink_goal_pending++;
+      p->reconfiguring = 2;
+    }
   }
 }
 
 static void
 proto_rethink_goal(struct proto *p)
 {
+  int goal_pending = (p->reconfiguring == 2);
+
   if (p->reconfiguring && !p->active)
   {
     struct proto_config *nc = p->cf_new;
@@ -1609,7 +1653,8 @@ proto_rethink_goal(struct proto *p)
     mb_free(p->message);
     mb_free(p);
     if (!nc)
-      return;
+      goto done;
+
     p = proto_init(nc, after);
   }
 
@@ -1621,6 +1666,15 @@ proto_rethink_goal(struct proto *p)
   }
   else if (!p->active)
     proto_start(p);
+
+done:
+  if (goal_pending && !--proto_rethink_goal_pending)
+    protos_do_commit(
+	protos_commit_request.new,
+	protos_commit_request.old,
+	protos_commit_request.force_reconfig,
+	protos_commit_request.type
+	);
 }
 
 struct proto *

nest/protocol.h

@@ -40,12 +40,20 @@ struct symbol;
  */
 
 
+enum protocol_startup {
+  PROTOCOL_STARTUP_REGULAR = 0,		/* Regular network routing protocol, start last */
+  PROTOCOL_STARTUP_GENERATOR = 1,	/* Static route generator, start ahead of regulars */
+  PROTOCOL_STARTUP_CONNECTOR = 2,	/* Data connector, start first */
+  PROTOCOL_STARTUP_NECESSARY = 3,	/* Vital auxiliary data, start zeroth */
+};
+
 struct protocol {
   node n;
   char *name;
   char *template;			/* Template for automatic generation of names */
   int name_counter;			/* Counter for automatic name generation */
   uint preference;			/* Default protocol preference */
+  enum protocol_startup startup;	/* When to start / stop this protocol */
   uint channel_mask;			/* Mask of accepted channel types (NB_*) */
   uint proto_size;			/* Size of protocol data structure */
   uint config_size;			/* Size of protocol config data structure */

proto/pipe/pipe.c

@@ -334,6 +334,7 @@ struct protocol proto_pipe = {
   .template =		"pipe%d",
   .proto_size =		sizeof(struct pipe_proto),
   .config_size =	sizeof(struct pipe_config),
+  .startup =		PROTOCOL_STARTUP_CONNECTOR,
   .postconfig =		pipe_postconfig,
   .init =		pipe_init,
   .reconfigure =	pipe_reconfigure,

proto/rpki/rpki.c

@@ -994,6 +994,7 @@ struct protocol proto_rpki = {
   .preference = 	DEF_PREF_RPKI,
   .proto_size = 	sizeof(struct rpki_proto),
   .config_size =	sizeof(struct rpki_config),
+  .startup =		PROTOCOL_STARTUP_GENERATOR,
   .init = 		rpki_init,
   .start = 		rpki_start,
   .postconfig = 	rpki_postconfig,

proto/static/static.c

@@ -775,6 +775,7 @@ struct protocol proto_static = {
   .channel_mask =	NB_ANY,
   .proto_size =		sizeof(struct static_proto),
   .config_size =	sizeof(struct static_config),
+  .startup =		PROTOCOL_STARTUP_GENERATOR,
   .postconfig =		static_postconfig,
   .init =		static_init,
   .dump =		static_dump,

sysdep/unix/krt.c

@@ -243,6 +243,7 @@ struct protocol proto_unix_iface = {
   .template = 		"device%d",
   .proto_size =		sizeof(struct kif_proto),
   .config_size =	sizeof(struct kif_config),
+  .startup =		PROTOCOL_STARTUP_NECESSARY,
   .preconfig =		kif_preconfig,
   .init =		kif_init,
   .start =		kif_start,
@@ -993,6 +994,7 @@ struct protocol proto_unix_kernel = {
   .channel_mask =	NB_IP | MAYBE_IP6_SADR | MAYBE_MPLS,
   .proto_size =		sizeof(struct krt_proto),
   .config_size =	sizeof(struct krt_config),
+  .startup =		PROTOCOL_STARTUP_CONNECTOR,
   .preconfig =		krt_preconfig,
   .postconfig =		krt_postconfig,
   .init =		krt_init,