From 74800729c06fcc8e6d170d11745480103edbe03b Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Mon, 18 Nov 2024 22:06:50 +0100
Subject: [PATCH] Proto: The active flag converted to actual fifth protocol
 state

What was PS_DOWN before, is now PS_DOWN and PS_FLUSH.
---
 nest/iface.c                  |  6 ++-
 nest/neighbor.c               |  2 +-
 nest/proto.c                  | 83 +++++++++++++++++------------------
 nest/protocol.h               | 12 +++--
 proto/aggregator/aggregator.c |  2 +-
 proto/babel/babel.c           |  2 +-
 proto/bfd/bfd.c               |  2 +-
 proto/bgp/bgp.c               | 13 +++---
 proto/bgp/packets.c           |  5 ++-
 proto/bmp/bmp.c               |  8 ++--
 proto/mrt/mrt.c               |  6 +--
 proto/ospf/ospf.c             |  4 +-
 proto/pipe/pipe.c             |  2 +-
 proto/radv/radv.c             |  2 +-
 proto/rip/rip.c               |  2 +-
 proto/rpki/packets.c          |  4 +-
 proto/rpki/rpki.c             |  8 ++--
 proto/static/static.c         |  2 +-
 sysdep/unix/krt.c             |  8 ++--
 19 files changed, 93 insertions(+), 80 deletions(-)

diff --git a/nest/iface.c b/nest/iface.c
index 897948ac..ec4c54c0 100644
--- a/nest/iface.c
+++ b/nest/iface.c
@@ -291,7 +291,8 @@ ifa_send_notify(struct iface_subscription *s, unsigned c, struct ifa *a)
   SKIP_BACK_DECLARE(struct proto, p, iface_sub, s);
 
   if (s->ifa_notify &&
-      (p->proto_state != PS_DOWN) &&
+      (p->proto_state != PS_DOWN_XX) &&
+      (p->proto_state != PS_FLUSH) &&
       (!p->vrf || if_in_vrf(a->iface, p->vrf)))
     {
       if (p->debug & D_IFACES)
@@ -333,7 +334,8 @@ if_send_notify(struct iface_subscription *s, unsigned c, struct iface *i)
   SKIP_BACK_DECLARE(struct proto, p, iface_sub, s);
 
   if (s->if_notify &&
-      (p->proto_state != PS_DOWN) &&
+      (p->proto_state != PS_DOWN_XX) &&
+      (p->proto_state != PS_FLUSH) &&
       (!p->vrf || if_in_vrf(i, p->vrf)))
     {
       if (p->debug & D_IFACES)
diff --git a/nest/neighbor.c b/nest/neighbor.c
index 257e61c4..a477235e 100644
--- a/nest/neighbor.c
+++ b/nest/neighbor.c
@@ -395,7 +395,7 @@ neigh_unlink_locked(neighbor *n)
   struct proto *p = n->proto;
   proto_neigh_rem_node(&p->neighbors, n);
 
-  if ((p->proto_state == PS_DOWN) && EMPTY_TLIST(proto_neigh, &p->neighbors))
+  if ((p->proto_state == PS_FLUSH) && EMPTY_TLIST(proto_neigh, &p->neighbors))
     proto_send_event(p, p->event);
 
   n->proto = NULL;
diff --git a/nest/proto.c b/nest/proto.c
index dee72eb2..12e06be8 100644
--- a/nest/proto.c
+++ b/nest/proto.c
@@ -68,7 +68,7 @@ static inline void channel_refeed(struct channel *c, struct rt_feeding_request *
 }
 
 static inline int proto_is_done(struct proto *p)
-{ return (p->proto_state == PS_DOWN) && proto_is_inactive(p); }
+{ return (p->proto_state == PS_FLUSH) && proto_is_inactive(p); }
 
 static inline int channel_is_active(struct channel *c)
 { return (c->channel_state != CS_DOWN); }
@@ -1219,7 +1219,7 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con
   {
     /* We could add the channel, but currently it would just stay in down state
        until protocol is restarted, so it is better to force restart anyways. */
-    if (p->proto_state != PS_DOWN)
+    if (p->proto_state != PS_DOWN_XX)
     {
       log(L_INFO "Cannot add channel %s.%s", p->name, cf->name);
       return 0;
@@ -1250,24 +1250,6 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con
   return 1;
 }
 
-
-static void
-proto_cleanup(struct proto *p)
-{
-  CALL(p->proto->cleanup, p);
-
-  if (p->pool)
-  {
-    rp_free(p->pool);
-    p->pool = NULL;
-  }
-
-  p->active = 0;
-  proto_log_state_change(p);
-
-  proto_rethink_goal(p);
-}
-
 static void
 proto_loop_stopped(void *ptr)
 {
@@ -1280,10 +1262,10 @@ proto_loop_stopped(void *ptr)
   birdloop_free(p->loop);
   p->loop = &main_birdloop;
 
-  proto_cleanup(p);
+  proto_notify_state(p, PS_DOWN_XX);
+  proto_rethink_goal(p);
 }
 
-
 static void
 proto_event(void *ptr)
 {
@@ -1307,7 +1289,10 @@ proto_event(void *ptr)
     if (p->loop != &main_birdloop)
       birdloop_stop_self(p->loop, proto_loop_stopped, p);
     else
-      proto_cleanup(p);
+    {
+      proto_notify_state(p, PS_DOWN_XX);
+      proto_rethink_goal(p);
+    }
   }
 }
 
@@ -1382,7 +1367,7 @@ proto_init(struct proto_config *c, struct proto *after)
   struct proto *p = pr->init(c);
 
   p->loop = &main_birdloop;
-  p->proto_state = PS_DOWN;
+  p->proto_state = PS_DOWN_XX;
   p->last_state_change = current_time();
   p->vrf = c->vrf;
   proto_add_after(&global_proto_list, p, after);
@@ -1565,7 +1550,7 @@ static int
 proto_reconfigure(struct proto *p, struct proto_config *oc, struct proto_config *nc, int type)
 {
   /* If the protocol is DOWN, we just restart it */
-  if (p->proto_state == PS_DOWN)
+  if (p->proto_state == PS_DOWN_XX)
     return 0;
 
   /* If there is a too big change in core attributes, ... */
@@ -1794,7 +1779,7 @@ proto_shutdown(struct proto *p)
     /* Going down */
     DBG("Kicking %s down\n", p->name);
     PD(p, "Shutting down");
-    proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN));
+    proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_FLUSH));
     if (p->reconfiguring)
     {
       proto_rethink_goal_pending++;
@@ -1808,7 +1793,7 @@ proto_rethink_goal(struct proto *p)
 {
   int goal_pending = (p->reconfiguring == 2);
 
-  if (p->reconfiguring && !p->active)
+  if (p->reconfiguring && (p->proto_state == PS_DOWN_XX))
   {
     struct proto_config *nc = p->cf_new;
     struct proto *after = p->n.prev;
@@ -1835,7 +1820,7 @@ proto_rethink_goal(struct proto *p)
     PROTO_LOCKED_FROM_MAIN(p)
       proto_shutdown(p);
   }
-  else if (!p->active)
+  else if (p->proto_state == PS_DOWN_XX)
     proto_start(p);
 
 done:
@@ -2049,9 +2034,9 @@ protos_dump_all(void)
   WALK_TLIST(proto, p, &global_proto_list) PROTO_LOCKED_FROM_MAIN(p)
   {
 #define DPF(x)	(p->x ? " " #x : "")
-    debug("  protocol %s (%p) state %s with %d active channels flags: %s%s%s%s\n",
+    debug("  protocol %s (%p) state %s with %d active channels flags: %s%s%s\n",
 	p->name, p, p_states[p->proto_state], p->active_channels,
-	DPF(disabled), DPF(active), DPF(do_stop), DPF(reconfiguring));
+	DPF(disabled), DPF(do_stop), DPF(reconfiguring));
 #undef DPF
 
     struct channel *c;
@@ -2070,7 +2055,9 @@ protos_dump_all(void)
     debug("\tSOURCES\n");
     rt_dump_sources(&p->sources);
 
-    if (p->proto->dump && (p->proto_state != PS_DOWN))
+    if (p->proto->dump &&
+	(p->proto_state != PS_DOWN_XX) &&
+	(p->proto_state != PS_FLUSH))
       p->proto->dump(p);
   }
 }
@@ -2357,8 +2344,6 @@ channel_reset_limit(struct channel *c, struct limit *l, int dir)
 static inline void
 proto_do_start(struct proto *p)
 {
-  p->active = 1;
-
   p->sources.debug = p->debug;
   rt_init_sources(&p->sources, p->name, proto_event_list(p));
 
@@ -2452,18 +2437,18 @@ proto_notify_state(struct proto *p, uint state)
   switch (state)
   {
   case PS_START:
-    ASSERT(ps == PS_DOWN || ps == PS_UP);
+    ASSERT(ps == PS_DOWN_XX || ps == PS_UP);
 
-    if (ps == PS_DOWN)
+    if (ps == PS_DOWN_XX)
       proto_do_start(p);
-    else
+    else 
       proto_do_pause(p);
     break;
 
   case PS_UP:
-    ASSERT(ps == PS_DOWN || ps == PS_START);
+    ASSERT(ps == PS_DOWN_XX || ps == PS_START);
 
-    if (ps == PS_DOWN)
+    if (ps == PS_DOWN_XX)
       proto_do_start(p);
 
     proto_do_up(p);
@@ -2475,13 +2460,26 @@ proto_notify_state(struct proto *p, uint state)
     proto_do_stop(p);
     break;
 
-  case PS_DOWN:
+  case PS_FLUSH:
     if (ps != PS_STOP)
       proto_do_stop(p);
 
     proto_do_down(p);
     break;
 
+  case PS_DOWN_XX:
+    ASSERT(ps == PS_FLUSH);
+
+    CALL(p->proto->cleanup, p);
+
+    if (p->pool)
+    {
+      rp_free(p->pool);
+      p->pool = NULL;
+    }
+
+    break;
+
   default:
     bug("%s: Invalid state %d", p->name, ps);
   }
@@ -2498,10 +2496,11 @@ proto_state_name(struct proto *p)
 {
   switch (p->proto_state)
   {
-  case PS_DOWN:		return p->active ? "flush" : "down";
+  case PS_DOWN_XX:		return "down";
   case PS_START:	return "start";
   case PS_UP:		return "up";
   case PS_STOP:		return "stop";
+  case PS_FLUSH:	return "flush";
   default:		return "???";
   }
 }
@@ -2864,7 +2863,7 @@ proto_get_named(struct symbol *sym, struct protocol *pr)
   {
     p = NULL;
     WALK_TLIST(proto, q, &global_proto_list)
-      if ((q->proto == pr) && (q->proto_state != PS_DOWN))
+      if ((q->proto == pr) && (q->proto_state != PS_DOWN_XX))
       {
 	if (p)
 	  cf_error("There are multiple %s protocols running", pr->name);
@@ -2904,7 +2903,7 @@ proto_iterate_named(struct symbol *sym, struct protocol *proto, struct proto *ol
 	p;
 	p = p->n.next)
     {
-      if ((p->proto == proto) && (p->proto_state != PS_DOWN))
+      if ((p->proto == proto) && (p->proto_state != PS_DOWN_XX))
       {
 	cli_separator(this_cli);
 	return p;
diff --git a/nest/protocol.h b/nest/protocol.h
index 84dd5570..dc9aa68d 100644
--- a/nest/protocol.h
+++ b/nest/protocol.h
@@ -163,7 +163,6 @@ struct proto {
   byte net_type;			/* Protocol network type (NET_*), 0 for undefined */
   byte disabled;			/* Manually disabled */
   byte proto_state;			/* Protocol state machine (PS_*, see below) */
-  byte active;				/* From PS_START to cleanup after PS_STOP */
   byte do_stop;				/* Stop actions are scheduled */
   byte reconfiguring;			/* We're shutting down due to reconfiguration */
   byte gr_recovery;			/* Protocol should participate in graceful restart recovery */
@@ -318,6 +317,9 @@ extern pool *proto_pool;
  *
  *		DOWN    ---->    START
  *		  ^		   |
+ *		  |		   |
+ *		FLUSH		   |
+ *		  ^		   |
  *		  |		   V
  *		STOP    <----     UP
  *
@@ -330,7 +332,10 @@ extern pool *proto_pool;
  *			connection breaks down or the core requests
  *			protocol to be terminated, it goes to STOP state.
  *		STOP	Protocol is disconnecting from the network.
- *			After it disconnects, it returns to DOWN state.
+ *			After it disconnects, it goes to FLUSH state.
+ *		FLUSH	Protocol is waiting for channels to stop
+ *			and routes and all other assets to flush.
+ *			When done, it goes to DOWN state.
  *
  *	In:	start()	Called in DOWN state to request protocol startup.
  *			Returns new state: either UP or START (in this
@@ -349,10 +354,11 @@ extern pool *proto_pool;
  *			shutdown).
  */
 
-#define PS_DOWN 0
+#define PS_DOWN_XX 0
 #define PS_START 1
 #define PS_UP 2
 #define PS_STOP 3
+#define PS_FLUSH 4
 
 void proto_notify_state(struct proto *p, unsigned state);
 
diff --git a/proto/aggregator/aggregator.c b/proto/aggregator/aggregator.c
index be8609c8..70dd1dac 100644
--- a/proto/aggregator/aggregator.c
+++ b/proto/aggregator/aggregator.c
@@ -418,7 +418,7 @@ aggregator_shutdown(struct proto *P)
   }
   HASH_WALK_END;
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
diff --git a/proto/babel/babel.c b/proto/babel/babel.c
index 002368a9..b9894fd4 100644
--- a/proto/babel/babel.c
+++ b/proto/babel/babel.c
@@ -2636,7 +2636,7 @@ babel_shutdown(struct proto *P)
     babel_remove_iface(p, ifa);
   }
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
diff --git a/proto/bfd/bfd.c b/proto/bfd/bfd.c
index 923381e9..488f756d 100644
--- a/proto/bfd/bfd.c
+++ b/proto/bfd/bfd.c
@@ -1171,7 +1171,7 @@ bfd_shutdown(struct proto *P)
 
   bfd_drop_requests(p);
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c
index 6dd12149..1fae5a42 100644
--- a/proto/bgp/bgp.c
+++ b/proto/bgp/bgp.c
@@ -595,7 +595,7 @@ bgp_down(struct bgp_proto *p)
   }
 
   BGP_TRACE(D_EVENTS, "Down");
-  proto_notify_state(&p->p, PS_DOWN);
+  proto_notify_state(&p->p, PS_FLUSH);
 }
 
 static void
@@ -1547,7 +1547,7 @@ bgp_neigh_notify(neighbor *n)
   if (n != p->neigh)
     return;
 
-  if ((ps == PS_DOWN) || (ps == PS_STOP))
+  if ((ps == PS_FLUSH) || (ps == PS_STOP))
     return;
 
   int prepare = (ps == PS_START) && (bgp_start_state(p) == BSS_PREPARE);
@@ -1748,7 +1748,7 @@ bgp_start_locked(void *_p)
     /* As we do not start yet, we can just disable protocol */
     p->p.disabled = 1;
     bgp_store_error(p, NULL, BE_MISC, BEM_INVALID_NEXT_HOP);
-    proto_notify_state(&p->p, PS_DOWN);
+    proto_notify_state(&p->p, PS_FLUSH);
     return;
   }
 
@@ -2647,9 +2647,12 @@ bgp_last_errmsg(struct bgp_proto *p)
 static const char *
 bgp_state_dsc(struct bgp_proto *p)
 {
-  if (p->p.proto_state == PS_DOWN)
+  if (p->p.proto_state == PS_DOWN_XX)
     return "Down";
 
+  if (p->p.proto_state == PS_FLUSH)
+    return "Flush";
+
   int state = MAX(p->incoming_conn.state, p->outgoing_conn.state);
   if ((state == BS_IDLE) && (bgp_start_state(p) >= BSS_CONNECT) && p->passive)
     return "Passive";
@@ -2665,7 +2668,7 @@ bgp_get_status(struct proto *P, byte *buf)
   const char *err1 = bgp_err_classes[p->last_error_class];
   const char *err2 = bgp_last_errmsg(p);
 
-  if (P->proto_state == PS_DOWN)
+  if (P->proto_state == PS_DOWN_XX)
     bsprintf(buf, "%s%s", err1, err2);
   else
     bsprintf(buf, "%-14s%s%s", bgp_state_dsc(p), err1, err2);
diff --git a/proto/bgp/packets.c b/proto/bgp/packets.c
index 49d9e71d..2ad75c82 100644
--- a/proto/bgp/packets.c
+++ b/proto/bgp/packets.c
@@ -1241,7 +1241,8 @@ bgp_use_next_hop(struct bgp_export_state *s, eattr *a)
     return 1;
 
   /* Keep it when forwarded between single-hop BGPs on the same iface */
-  struct iface *ifa = (s->src && s->src->neigh && (s->src->p.proto_state != PS_DOWN)) ?
+  struct iface *ifa = (s->src && s->src->neigh &&
+      ((s->src->p.proto_state == PS_UP) || (s->src->p.proto_state == PS_START))) ?
     s->src->neigh->iface : NULL;
   return p->neigh && (p->neigh->iface == ifa);
 }
@@ -3503,7 +3504,7 @@ bgp_do_uncork(callback *cb)
   ASSERT_DIE(birdloop_inside(p->p.loop));
   ASSERT_DIE(p->p.active_loops--);
 
-  if (p->p.proto_state == PS_DOWN)
+  if (p->p.proto_state == PS_FLUSH)
     ev_send_loop(p->p.loop, p->p.event);
   else if (p->conn && (p->conn->state == BS_ESTABLISHED) && !p->conn->sk->rx_hook)
   {
diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 261e9fdd..6187914d 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -1281,7 +1281,7 @@ bmp_shutdown(struct proto *P)
   p->sock_err = 0;
   rem_node(&p->bmp_node);
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
@@ -1314,8 +1314,10 @@ bmp_get_status(struct proto *P, byte *buf)
 {
   struct bmp_proto *p = (void *) P;
 
-  if (P->proto_state == PS_DOWN)
+  if (P->proto_state == PS_DOWN_XX)
     bsprintf(buf, "Down");
+  else if (P->proto_state == PS_FLUSH)
+    bsprintf(buf, "Flush");
   else
   {
     const char *state = !p->started ? (!p->sk ? "Idle" : "Connect") : "Established";
@@ -1332,7 +1334,7 @@ bmp_show_proto_info(struct proto *P)
 {
   struct bmp_proto *p = (void *) P;
 
-  if (P->proto_state != PS_DOWN)
+  if (P->proto_state != PS_DOWN_XX)
   {
     cli_msg(-1006, "  %-19s %I", "Station address:", p->station_ip);
     cli_msg(-1006, "  %-19s %u", "Station port:", p->station_port);
diff --git a/proto/mrt/mrt.c b/proto/mrt/mrt.c
index bab2dc30..31e46d91 100644
--- a/proto/mrt/mrt.c
+++ b/proto/mrt/mrt.c
@@ -345,7 +345,7 @@ mrt_peer_table_dump(struct mrt_table_dump_state *s)
 
       struct protocol *type = (struct protocol *)ea_get_ptr(eal, &ea_protocol_type, 0);
       int state = ea_get_int(eal, &ea_state, 0);
-      if ((type == &proto_bgp) && (state != PS_DOWN))
+      if ((type == &proto_bgp) && (state != PS_DOWN_XX))
       {
         int rem_id = ea_get_int(eal, &ea_bgp_rem_id, 0);
         int rem_as = ea_get_int(eal, &ea_bgp_rem_as, 0);
@@ -729,7 +729,7 @@ mrt_proto_dump_done(struct mrt_table_dump_state *s)
   if (p->p.proto_state == PS_STOP)
   {
     mrt_free_table_list(p->table_list, p->table_list_len);
-    proto_notify_state(&p->p, PS_DOWN);
+    proto_notify_state(&p->p, PS_FLUSH);
   }
 }
 
@@ -966,7 +966,7 @@ mrt_shutdown(struct proto *P)
     return PS_STOP;
 
   mrt_free_table_list(p->table_list, p->table_list_len);
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
diff --git a/proto/ospf/ospf.c b/proto/ospf/ospf.c
index 7c649214..d6cfb4ec 100644
--- a/proto/ospf/ospf.c
+++ b/proto/ospf/ospf.c
@@ -541,7 +541,7 @@ ospf_shutdown(struct proto *P)
   }
   FIB_WALK_END;
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static void
@@ -549,7 +549,7 @@ ospf_get_status(struct proto *P, byte * buf)
 {
   struct ospf_proto *p = (struct ospf_proto *) P;
 
-  if (p->p.proto_state == PS_DOWN)
+  if ((p->p.proto_state == PS_DOWN_XX) || (p->p.proto_state == PS_FLUSH))
     buf[0] = 0;
   else
   {
diff --git a/proto/pipe/pipe.c b/proto/pipe/pipe.c
index bf3b89f0..4e9f8d8e 100644
--- a/proto/pipe/pipe.c
+++ b/proto/pipe/pipe.c
@@ -278,7 +278,7 @@ pipe_show_proto_info(struct proto *P)
   channel_show_limit(&p->sec->in_limit, "Export limit:",
       (p->sec->limit_active & (1 << PLD_IN)), p->sec->limit_actions[PLD_IN]);
 
-  if (P->proto_state != PS_DOWN)
+  if (P->proto_state != PS_DOWN_XX)
     pipe_show_stats(p);
 }
 
diff --git a/proto/radv/radv.c b/proto/radv/radv.c
index ea0b8d38..f5745653 100644
--- a/proto/radv/radv.c
+++ b/proto/radv/radv.c
@@ -643,7 +643,7 @@ radv_shutdown(struct proto *P)
   WALK_LIST(ifa, p->iface_list)
     radv_iface_shutdown(ifa);
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
diff --git a/proto/rip/rip.c b/proto/rip/rip.c
index f62ec667..22389e33 100644
--- a/proto/rip/rip.c
+++ b/proto/rip/rip.c
@@ -1213,7 +1213,7 @@ rip_shutdown(struct proto *P)
   WALK_LIST_FIRST(ifa, p->iface_list)
     rip_remove_iface(p, ifa);
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static int
diff --git a/proto/rpki/packets.c b/proto/rpki/packets.c
index d7895a22..07847db0 100644
--- a/proto/rpki/packets.c
+++ b/proto/rpki/packets.c
@@ -909,7 +909,7 @@ rpki_rx_hook(struct birdsock *sk, uint size)
   struct rpki_cache *cache = sk->data;
   struct rpki_proto *p = cache->p;
 
-  if ((p->p.proto_state == PS_DOWN) || (p->cache != cache))
+  if ((p->p.proto_state == PS_FLUSH) || (p->cache != cache))
     return 0;
 
   byte *pkt_start = sk->rbuf;
@@ -934,7 +934,7 @@ rpki_rx_hook(struct birdsock *sk, uint size)
     rpki_rx_packet(cache, pdu);
 
     /* It is possible that bird socket was freed/closed */
-    if (p->p.proto_state == PS_DOWN || sk != cache->tr_sock->sk)
+    if (p->p.proto_state == PS_FLUSH || sk != cache->tr_sock->sk)
       return 0;
 
     pkt_start += pdu_size;
diff --git a/proto/rpki/rpki.c b/proto/rpki/rpki.c
index 9c494f51..4c19321f 100644
--- a/proto/rpki/rpki.c
+++ b/proto/rpki/rpki.c
@@ -225,7 +225,7 @@ rpki_force_restart_proto(struct rpki_proto *p)
   /* Sign as freed */
   p->cache = NULL;
 
-  proto_notify_state(&p->p, PS_DOWN);
+  proto_notify_state(&p->p, PS_FLUSH);
 }
 
 /**
@@ -676,7 +676,7 @@ rpki_shutdown(struct proto *P)
   rpki_force_restart_proto(p);
 
   /* Protocol memory pool will be automatically freed */
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 
@@ -839,7 +839,7 @@ rpki_get_status(struct proto *P, byte *buf)
 {
   struct rpki_proto *p = (struct rpki_proto *) P;
 
-  if (P->proto_state == PS_DOWN)
+  if ((P->proto_state == PS_DOWN_XX) || (P->proto_state == PS_FLUSH))
   {
     *buf = 0;
     return;
@@ -867,7 +867,7 @@ rpki_show_proto_info(struct proto *P)
   struct rpki_config *cf = (void *) p->p.cf;
   struct rpki_cache *cache = p->cache;
 
-  if (P->proto_state == PS_DOWN)
+  if ((P->proto_state == PS_DOWN_XX) || (P->proto_state == PS_FLUSH))
     return;
 
   if (cache)
diff --git a/proto/static/static.c b/proto/static/static.c
index d091924b..cd393dd7 100644
--- a/proto/static/static.c
+++ b/proto/static/static.c
@@ -570,7 +570,7 @@ static_shutdown(struct proto *P)
   WALK_LIST(r, cf->routes)
     static_reset_rte(p, r);
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static void
diff --git a/sysdep/unix/krt.c b/sysdep/unix/krt.c
index cb1093ef..2770b8be 100644
--- a/sysdep/unix/krt.c
+++ b/sysdep/unix/krt.c
@@ -160,7 +160,7 @@ kif_shutdown(struct proto *P)
   kif_sys_shutdown(p);
   kif_proto = NULL;
 
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static void
@@ -876,7 +876,7 @@ krt_shutdown(struct proto *P)
   krt_scan_timer_stop(p);
 
   if (p->p.proto_state == PS_START)
-    return PS_DOWN;
+    return PS_FLUSH;
 
   /* FIXME we should flush routes even when persist during reconfiguration */
   if (p->initialized && !KRT_CF->persist && (P->down_code != PDC_CMD_GR_DOWN))
@@ -903,12 +903,12 @@ krt_shutdown(struct proto *P)
 
     krt_do_scan(p);
     krt_cleanup(p);
-    proto_notify_state(&p->p, PS_DOWN);
+    proto_notify_state(&p->p, PS_FLUSH);
     rt_feeder_unsubscribe(&req);
   }
   else
     krt_cleanup(p);
-  return PS_DOWN;
+  return PS_FLUSH;
 }
 
 static void