From c48a7ac0cc76d2d5c1682251cd8b03a8953bd6b5 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Tue, 17 Sep 2024 15:59:07 +0200
Subject: [PATCH 01/15] BMP: drop an unused mempool

---
 proto/bmp/bmp.c | 1 -
 proto/bmp/bmp.h | 1 -
 2 files changed, 2 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 261e9fdd..599024f5 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -1244,7 +1244,6 @@ bmp_start(struct proto *P)
   struct bmp_proto *p = (void *) P;
 
   p->buffer_mpool = rp_new(P->pool, "BMP Buffer");
-  p->map_mem_pool = rp_new(P->pool, "BMP Map");
   p->tx_mem_pool = rp_new(P->pool, "BMP Tx");
   p->update_msg_mem_pool = rp_new(P->pool, "BMP Update");
   p->tx_ev = ev_new_init(p->p.pool, bmp_fire_tx, p);
diff --git a/proto/bmp/bmp.h b/proto/bmp/bmp.h
index d69aaafb..45844836 100644
--- a/proto/bmp/bmp.h
+++ b/proto/bmp/bmp.h
@@ -65,7 +65,6 @@ struct bmp_proto {
   // Below fields are for internal use
   // struct bmp_peer_map bgp_peers;   // Stores 'bgp_proto' structure per BGP peer
   pool *buffer_mpool;              // Memory pool used for BMP buffer allocations
-  pool *map_mem_pool;              // Memory pool used for BMP map allocations
   pool *tx_mem_pool;               // Memory pool used for packet allocations designated to BMP collector
   pool *update_msg_mem_pool;       // Memory pool used for BPG UPDATE MSG allocations
   list tx_queue;                   // Stores queued packets going to be sent

From 460321cfe979459e3b78ba87694f29865d321612 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Thu, 28 Nov 2024 23:08:50 +0100
Subject: [PATCH 02/15] Formatting numbers with order prefixes

Unit tests by Ondrej Zajicek.
---
 lib/printf.c      | 121 +++++++++++++++++++++++++++++++++++++++++++++-
 lib/printf_test.c |  63 ++++++++++++++++++++++++
 lib/string.h      |   4 ++
 3 files changed, 187 insertions(+), 1 deletion(-)

diff --git a/lib/printf.c b/lib/printf.c
index 68d3bb74..318e683c 100644
--- a/lib/printf.c
+++ b/lib/printf.c
@@ -8,7 +8,8 @@
  */
 
 #include "nest/bird.h"
-#include "string.h"
+#include "lib/macro.h"
+#include "lib/string.h"
 
 #include <errno.h>
 
@@ -631,3 +632,121 @@ char *lp_sprintf(linpool *p, const char *fmt, ...)
   va_end(args);
   return out;
 }
+
+static const u64 decadic_multiplier[] = {
+		  1,		     10,		 100,		      1000,
+	      10000,		 100000,	     1000000,		  10000000,
+#if 0
+	  100000000,	     1000000000,	 10000000000,	      100000000000,
+      1000000000000,	 10000000000000,     100000000000000,	  1000000000000000,
+  10000000000000000, 100000000000000000, 1000000000000000000,
+#endif
+};
+
+static const u64 decmul_limit[] = {
+  ~0ULL /		    1, ~0ULL /		       10,
+  ~0ULL /		  100, ~0ULL /		     1000,
+  ~0ULL /		10000, ~0ULL /		   100000,
+  ~0ULL /	      1000000, ~0ULL /		 10000000,
+#if 0
+  ~0ULL /	    100000000, ~0ULL /         1000000000,
+  ~0ULL /	  10000000000, ~0ULL /	     100000000000,
+  ~0ULL /	1000000000000, ~0ULL /     10000000000000,
+  ~0ULL /     100000000000000, ~0ULL /   1000000000000000,
+  ~0ULL /   10000000000000000, ~0ULL / 100000000000000000,
+  ~0ULL / 1000000000000000000,
+#endif
+};
+
+STATIC_ASSERT(sizeof decadic_multiplier == sizeof decmul_limit);
+
+char *fmt_order(u64 value, uint decimals, u64 kb_threshold)
+{
+  bool too_big = (value + 512 < 512ULL);
+
+  u64 mv = value;
+  uint magnitude = 0;
+  while (mv > kb_threshold)
+  {
+    magnitude++;
+    mv = (mv + (too_big ? 0 : 512)) / 1024;
+  }
+
+  uint shift = magnitude * 10;
+
+  /* The trivial case */
+  if (magnitude == 0)
+    return tmp_sprintf("%lu  ", value);
+
+  /* Now we can find the suffix and the main divisor */
+  ASSERT_DIE(magnitude < 7);
+  char suffix = " kMGTPE"[magnitude];
+
+  /* The value before the dot is available just by dividing */
+  u64 before_dot = value >> shift;
+
+  /* Remainder is more tricky. First we need to know it. */
+  u64 remainder = value - (before_dot << shift);
+
+  /* We would like to compute (remainder * decmul) / divisor
+   * in integers but it's tricky because of u64 limits. */
+  ASSERT_DIE(decimals < ARRAY_SIZE(decadic_multiplier));
+  u64 decmul = decadic_multiplier[decimals];
+  u64 product;
+
+  if (remainder < decmul_limit[decimals])
+  {
+    /* The easier version: Everything fits into u64 */
+    product = remainder * decmul;
+    product >>= shift - 1;
+    product++;
+    product >>= 1;
+  }
+  else
+  {
+    /* Harder version: We have to multiply by parts.
+     * Fortunately, decmul always fits into 32 bits. */
+    /* After this, product = lower + (upper << 32). */
+    u64 lower = (remainder & ((1ULL << 32) - 1)) * decmul;
+    u64 upper = (remainder >> 32) * decmul;
+
+    if (shift < 33)
+    {
+      /* Divide lower */
+      lower >>= shift - 1;
+
+      /* Add the full upper part, not shifted enough to lose any bits */
+      lower += upper << (33 - shift);
+    }
+    else
+    {
+      /* First move the shifted-out bits from upper to lower */
+      lower += (upper & ((1ULL << (shift - 32)) - 1)) << 32;
+
+      /* Then we can divide */
+      lower >>= shift - 1;
+
+      /* And add the shifted upper part */
+      lower += upper >> (shift - 33);
+    }
+
+    /* Now we finish the division by rounding */
+    product = (lower + 1) >> 1;
+  }
+
+  if (product == decmul)
+  {
+    product = 0;
+    before_dot++;
+  }
+
+  ASSERT_DIE(product < decmul);
+
+  /* And now we finally have all the numbers to print! */
+  if (decimals)
+    return tmp_sprintf("%lu.%0*lu %c",
+	before_dot, decimals, product, suffix
+	);
+  else
+    return tmp_sprintf("%lu %c", before_dot, suffix);
+}
diff --git a/lib/printf_test.c b/lib/printf_test.c
index 996f34ae..b3d4d560 100644
--- a/lib/printf_test.c
+++ b/lib/printf_test.c
@@ -117,6 +117,68 @@ t_bstrcmp(void)
   return 1;
 }
 
+static int
+t_fmt_order(void)
+{
+  struct fmt_order_tv {
+    u64 value;
+    int decimals;
+    u64 threshold;
+    const char *expect;
+  } test_vectors [] = {
+    { 9999, 1, 10000, "9999  " },
+    { 10001, 1, 10000, "9.8 k" },
+    { 10001, 2, 10000, "9.77 k" },
+    { 10001, 3, 10000, "9.767 k" },
+    { 1048575, 0, 10000, "1024 k" },
+    { 1048575, 1, 10000, "1024.0 k" },
+    { 1048575, 2, 10000, "1024.00 k" },
+    { 1048575, 3, 10000, "1023.999 k" },
+    { 1048575, 4, 10000, "1023.9990 k" },
+    { 1048575, 5, 10000, "1023.99902 k" },
+    { 1048575, 6, 10000, "1023.999023 k" },
+    { 1048575, 0, 1000, "1 M" },
+    { 1048575, 1, 1000, "1.0 M" },
+    { 1048575, 2, 1000, "1.00 M" },
+    { 1048575, 3, 1000, "1.000 M" },
+    { 1048575, 4, 1000, "1.0000 M" },
+    { 1048575, 5, 1000, "1.00000 M" },
+    { 1048575, 6, 1000, "0.999999 M" },
+    { 1048577, 6, 10000, "1024.000977 k" },
+    { 1048577, 6, 1000, "1.000001 M" },
+    { 1048577, 6, 100, "1.000001 M" },
+    { 1048577, 6, 10, "1.000001 M" },
+    { 1048577, 6, 1, "1.000001 M" },
+    { 10000000000000, 6, 10000, "9313.225746 G" },
+    { 10000000000000, 6, 1000, "9.094947 T" },
+    { 123456789123456789, 0, 1000, "110 P" },
+    { 123456789123456789, 4, 1000, "109.6517 P" },
+    { 123456789123456789, 7, 1000, "109.6516559 P" },
+    { 1234567890123456789, 0, 1000, "1 E" },
+    { 1234567890123456789, 1, 1000, "1.1 E" },
+    { 1234567890123456789, 2, 1000, "1.07 E" },
+    { 1234567890123456789, 3, 1000, "1.071 E" },
+    { 1234567890123456789, 4, 1000, "1.0708 E" },
+    { 1234567890123456789, 5, 1000, "1.07082 E" },
+    { 1234567890123456789, 6, 1000, "1.070817 E" },
+    { 1234567890123456789, 7, 1000, "1.0708170 E" },
+    { 9444732965739290427U, 3, 1000, "8.192 E" },
+    { 9444732965739290427U, 6, 1000, "8.192000 E" },
+    { 18446744073709551615U, 2, 1000, "16.00 E" },
+  };
+
+  for (int i = 0; i < (int)ARRAY_SIZE(test_vectors); i++)
+  {
+    const char *result = fmt_order(test_vectors[i].value, test_vectors[i].decimals, test_vectors[i].threshold);
+    const char *expect = test_vectors[i].expect;
+
+    bt_assert_msg(strncmp(result, expect, strlen(expect)) == 0,
+		  "case %d, result \"%s\", expect \"%s\"", i, result, expect);
+  }
+
+  return 1;
+}
+
 int
 main(int argc, char *argv[])
 {
@@ -126,6 +188,7 @@ main(int argc, char *argv[])
   bt_test_suite(t_router_id, "print router id");
   bt_test_suite(t_time, "print time");
   bt_test_suite(t_bstrcmp, "bstrcmp");
+  bt_test_suite(t_fmt_order, "fmt_order");
 
   return bt_exit_value();
 }
diff --git a/lib/string.h b/lib/string.h
index 0c21f513..8831666c 100644
--- a/lib/string.h
+++ b/lib/string.h
@@ -24,6 +24,8 @@ char *mb_sprintf(pool *p, const char *fmt, ...);
 char *mb_vsprintf(pool *p, const char *fmt, va_list args);
 char *lp_sprintf(linpool *p, const char *fmt, ...);
 char *lp_vsprintf(linpool *p, const char *fmt, va_list args);
+#define tmp_sprintf(...)    lp_sprintf(tmp_linpool, __VA_ARGS__)
+#define tmp_vsprintf(...)   lp_vsprintf(tmp_linpool, __VA_ARGS__)
 
 int buffer_vprint(buffer *buf, const char *fmt, va_list args);
 int buffer_print(buffer *buf, const char *fmt, ...);
@@ -33,6 +35,8 @@ u64 bstrtoul10(const char *str, char **end);
 u64 bstrtoul16(const char *str, char **end);
 byte bstrtobyte16(const char *str);
 
+char *fmt_order(u64 value, uint decimals, u64 kb_threshold);
+
 int bstrhextobin(const char *s, byte *b);
 int bstrbintohex(const byte *b, size_t len, char *buf, size_t size, char delim) ACCESS_READ(1, 2) ACCESS_WRITE(3, 4);
 

From e6a100b31a7637ee739338e4b933367707ec931f Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Tue, 17 Sep 2024 16:27:54 +0200
Subject: [PATCH 03/15] BMP: simplified update queuing and better memory
 performance

This commit is quite a substantial rework of the underlying layers in
BMP TX:

- several unnecessary layers of indirection dropped, including most of
  the original BMP's buffer machinery
- all messages are now written directly into one protocol's buffer
  allocated for the whole time big enough to fit every possible message
- output blocks are allocated by pages and immediately returned when
  used, improving the overall memory footprint
- no intermediary allocation is done from the heap altogether
- there is a documented and configurable limit on the TX queue size
---
 doc/bird.sgml       |  12 ++
 proto/bgp/bgp.h     |   2 +-
 proto/bgp/packets.c |   8 +-
 proto/bmp/Makefile  |   4 +-
 proto/bmp/bmp.c     | 503 +++++++++++++++++++++++++++-----------------
 proto/bmp/bmp.h     |  25 ++-
 proto/bmp/buffer.c  |  65 ------
 proto/bmp/buffer.h  |  77 -------
 proto/bmp/config.Y  |   5 +
 9 files changed, 355 insertions(+), 346 deletions(-)
 delete mode 100644 proto/bmp/buffer.c
 delete mode 100644 proto/bmp/buffer.h

diff --git a/doc/bird.sgml b/doc/bird.sgml
index d1375caf..b6b77691 100644
--- a/doc/bird.sgml
+++ b/doc/bird.sgml
@@ -3808,6 +3808,15 @@ by default and have to be enabled during installation by the configure option
 routes (in <ref id="bgp-import-table" name="BGP import tables">) and post-policy
 routes (in regular routing tables). All BGP protocols are monitored automatically.
 
+<sect1>Configuration (incomplete)
+<label id="bmp-config">
+
+<p><descrip>
+	<tag><label id="bmp-tx-buffer-limit">tx buffer limit <m/number/</tag>
+	How much data we are going to queue before we call the session stuck
+	and restart it, in megabytes. Default value: 1024 (effectively 1 gigabyte).
+</descrip>
+
 <sect1>Example
 <label id="bmp-exam">
 
@@ -3821,6 +3830,9 @@ protocol bmp {
 
 	# Monitor accepted routes (passed import filters)
 	monitoring rib in post_policy;
+
+	# Allow only 64M of pending data
+	tx buffer limit 64;
 }
 </code>
 
diff --git a/proto/bgp/bgp.h b/proto/bgp/bgp.h
index 7127bc88..4cef8caf 100644
--- a/proto/bgp/bgp.h
+++ b/proto/bgp/bgp.h
@@ -650,7 +650,7 @@ int bgp_get_attr(const struct eattr *e, byte *buf, int buflen);
 void bgp_get_route_info(struct rte *, byte *buf);
 int bgp_total_aigp_metric_(rte *e, u64 *metric, const struct adata **ad);
 
-byte * bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, const net_addr *n, const struct rte *new, const struct rte_src *src);
+byte * bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, byte *end, const net_addr *n, const struct rte *new, const struct rte_src *src);
 
 #define BGP_AIGP_METRIC		1
 #define BGP_AIGP_MAX		U64(0xffffffffffffffff)
diff --git a/proto/bgp/packets.c b/proto/bgp/packets.c
index 30da3294..ec648231 100644
--- a/proto/bgp/packets.c
+++ b/proto/bgp/packets.c
@@ -2482,12 +2482,10 @@ bgp_create_mp_unreach(struct bgp_write_state *s, struct bgp_bucket *buck, byte *
 #ifdef CONFIG_BMP
 
 static byte *
-bgp_create_update_bmp(struct bgp_channel *c, byte *buf, struct bgp_bucket *buck, bool update)
+bgp_create_update_bmp(struct bgp_channel *c, byte *buf, byte *end, struct bgp_bucket *buck, bool update)
 {
   struct bgp_proto *p = (void *) c->c.proto;
-  byte *end = buf + (BGP_MAX_EXT_MSG_LENGTH - BGP_HEADER_LENGTH);
   byte *res = NULL;
-  /* FIXME: must be a bit shorter */
 
   struct lp_state tmpp;
   lp_save(tmp_linpool, &tmpp);
@@ -2535,7 +2533,7 @@ bgp_bmp_prepare_bgp_hdr(byte *buf, const u16 msg_size, const u8 msg_type)
 }
 
 byte *
-bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, const net_addr *n,
+bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, byte *end, const net_addr *n,
 		   const struct rte *new, const struct rte_src *src)
 {
 //  struct bgp_proto *p = (void *) c->c.proto;
@@ -2561,7 +2559,7 @@ bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, const net_addr *n,
   net_copy(px->net, n);
   add_tail(&b->prefixes, &px->buck_node);
 
-  byte *end = bgp_create_update_bmp(c, pkt, b, !!new);
+  end = bgp_create_update_bmp(c, pkt, end, b, !!new);
 
   if (end)
     bgp_bmp_prepare_bgp_hdr(buf, end - buf, PKT_UPDATE);
diff --git a/proto/bmp/Makefile b/proto/bmp/Makefile
index d6fca1aa..e34f3822 100644
--- a/proto/bmp/Makefile
+++ b/proto/bmp/Makefile
@@ -1,6 +1,6 @@
-src := bmp.c buffer.c map.c
+src := bmp.c map.c
 obj := $(src-o-files)
 $(all-daemon)
 $(cf-local)
 
-tests_objs := $(tests_objs) $(src-o-files)
\ No newline at end of file
+tests_objs := $(tests_objs) $(src-o-files)
diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 599024f5..6dcf1562 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -29,7 +29,6 @@
  */
 
 #include "proto/bmp/bmp.h"
-#include "proto/bmp/buffer.h"
 #include "proto/bmp/map.h"
 
 #include <sys/socket.h>
@@ -94,16 +93,10 @@ enum bmp_message_type {
   BMP_ROUTE_MIRROR_MSG = 6 // Route Mirroring Message
 };
 
-// Total size of Common Header
-#define BMP_COMMON_HDR_SIZE 6
 // Defines size of padding when IPv4 address is going to be put into field
 // which can accept also IPv6 address
 #define BMP_PADDING_IP4_ADDR_SIZE 12
 
-/* BMP Per-Peer Header [RFC 7854 - Section 4.2] */
-// Total size of Per-Peer Header
-#define BMP_PER_PEER_HDR_SIZE 42
-
 enum bmp_peer_type {
   BMP_PEER_TYPE_GLOBAL_INSTANCE = 0,
   BMP_PEER_TYPE_RD_INSTANCE = 1,
@@ -166,6 +159,15 @@ enum bmp_peer_down_notif_reason {
   BMP_PEER_DOWN_REASON_PEER_DE_CONFIGURED = 5
 };
 
+struct bmp_peer_down_info {
+  u8 reason;
+  u8 fsm_code;
+  u8 err_code;
+  u8 err_subcode;
+  const byte *data;
+  int length;
+};
+
 /* BMP Termination Message [RFC 7854 - Section 4.5] */
 #define BMP_TERM_INFO_TYPE_SIZE 2
 enum bmp_term_info_type {
@@ -198,21 +200,42 @@ enum bmp_term_reason {
 #define IP4_MAX_TTL 255
 
 
-#define IF_COND_TRUE_PRINT_ERR_MSG_AND_RETURN_OPT_VAL(expr, msg, rv...)     \
-  do {                                                                      \
-    if ((expr))                                                             \
-    {                                                                       \
-      log(L_WARN "[BMP] " msg);                                             \
-      return rv;                                                            \
-    }                                                                       \
-  } while (0)
+#define bmp_buffer_need(b, sz)  ASSERT_DIE((b)->pos + (sz) <= (b)->end)
 
+// Idea for following macros has been taken from |proto/mrt/mrt.c|
+#define BMP_DEFINE_PUT_FUNC(S, T)                               \
+  static inline void                                            \
+  bmp_put_##S(buffer *b, const T x)                             \
+  {                                                             \
+    bmp_buffer_need(b, sizeof(T));			   \
+    put_##S(b->pos, x);                                    \
+    b->pos += sizeof(T);                                   \
+  }
 
-#define IF_PTR_IS_NULL_PRINT_ERR_MSG_AND_RETURN_OPT_VAL(p, msg, rv...)	\
-  do {									\
-    IF_COND_TRUE_PRINT_ERR_MSG_AND_RETURN_OPT_VAL(!(p), msg, rv);	\
-  } while (0)
+BMP_DEFINE_PUT_FUNC(u8, u8)
+BMP_DEFINE_PUT_FUNC(u16, u16)
+BMP_DEFINE_PUT_FUNC(u32, u32)
+BMP_DEFINE_PUT_FUNC(u64, u64)
+BMP_DEFINE_PUT_FUNC(ip4, ip4_addr)
+BMP_DEFINE_PUT_FUNC(ip6, ip6_addr)
 
+static inline void
+bmp_put_data(buffer *b, const void *src, const size_t len)
+{
+  ASSERT_DIE(b->pos + len <= b->end);
+  memcpy(b->pos, src, len);
+  b->pos += len;
+}
+
+static inline buffer
+bmp_default_buffer(struct bmp_proto *p)
+{
+  return (buffer) {
+    .start = p->msgbuf,
+    .pos = p->msgbuf,
+    .end = p->msgbuf + sizeof p->msgbuf,
+  };
+}
 
 static void bmp_connected(struct birdsock *sk);
 static void bmp_sock_err(sock *sk, int err);
@@ -225,18 +248,44 @@ bmp_send_peer_up_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
 
 static void bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_stream *bs);
 
-// Stores necessary any data in list
-struct bmp_data_node {
-  node n;
-  byte *data;
-  size_t data_size;
+// Stores TX data
+struct bmp_tx_buffer {
+  struct bmp_tx_buffer *next;
+  byte *pos;
+  byte data[];
+};
 
-  u32 remote_as;
-  u32 remote_id;
-  ip_addr remote_ip;
-  btime timestamp;
-  bool global_peer;
-  bool policy;
+#define bmp_tx_remains(b) (((byte *) (b) + page_size) - (b)->pos)
+
+/* A dummy resource to accurately show memory pages allocated for pending TX */
+struct bmp_tx_resource {
+  resource r;
+  struct bmp_proto *p;
+};
+
+static void
+bmp_tx_resource_free(resource *r UNUSED) {}
+
+static void
+bmp_tx_resource_dump(resource *r UNUSED) {}
+
+static struct resmem
+bmp_tx_resource_memsize(resource *r)
+{
+  struct bmp_proto *p = SKIP_BACK(struct bmp_tx_resource, r, r)->p;
+
+  return (struct resmem) {
+    .effective = p->tx_pending_count * page_size,
+    .overhead = sizeof(struct bmp_tx_resource),
+  };
+}
+
+static struct resclass bmp_tx_resource_class = {
+  .name = "BMP TX buffers",
+  .size = sizeof(struct bmp_tx_resource),
+  .free = bmp_tx_resource_free,
+  .dump = bmp_tx_resource_dump,
+  .memsize = bmp_tx_resource_memsize,
 };
 
 static void
@@ -268,6 +317,7 @@ bmp_init_msg_serialize(buffer *stream, const char *sys_descr, const char *sys_na
   // We include MIB-II sysDescr and sysName in BMP INIT MSG so that's why
   // allocated 2x BMP_INFO_TLV_FIX_SIZE memory pool size
   const size_t data_size = (2 * BMP_INFO_TLV_FIX_SIZE) + sys_descr_len + sys_name_len;
+
   bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
   bmp_common_hdr_serialize(stream, BMP_INIT_MSG, data_size);
   bmp_info_tlv_hdr_serialize(stream, BMP_INFO_TLV_TYPE_SYS_DESCR, sys_descr);
@@ -275,21 +325,62 @@ bmp_init_msg_serialize(buffer *stream, const char *sys_descr, const char *sys_na
 }
 
 static void
-bmp_schedule_tx_packet(struct bmp_proto *p, const byte *payload, const size_t size)
+bmp_schedule_tx_packet(struct bmp_proto *p, const byte *payload, size_t size)
 {
   ASSERT(p->started);
 
-  struct bmp_data_node *tx_data = mb_alloc(p->tx_mem_pool, sizeof (struct bmp_data_node));
-  tx_data->data = mb_alloc(p->tx_mem_pool, size);
-  memcpy(tx_data->data, payload, size);
-  tx_data->data_size = size;
-  add_tail(&p->tx_queue, &tx_data->n);
-
-  if (sk_tx_buffer_empty(p->sk)
-      && !ev_active(p->tx_ev))
+  while (size)
   {
-    ev_schedule(p->tx_ev);
+    if (!p->tx_last || !bmp_tx_remains(p->tx_last))
+    {
+      if (p->tx_pending_count >= p->tx_pending_limit)
+	return ev_schedule(p->tx_overflow_event);
+
+      p->tx_pending_count++;
+
+      struct bmp_tx_buffer *btb = alloc_page();
+      btb->pos = btb->data;
+      btb->next = NULL;
+
+      if (p->tx_last)
+      {
+	ASSERT_DIE(!p->tx_last->next);
+	p->tx_last->next = btb;
+      }
+      else
+	ASSERT_DIE(p->tx_pending_count == 1);
+
+      p->tx_last = btb;
+
+      if (!p->tx_pending)
+	p->tx_pending = btb;
+    }
+
+    size_t cpylen = bmp_tx_remains(p->tx_last);
+    if (size < cpylen)
+      cpylen = size;
+
+    memcpy(p->tx_last->pos, payload, cpylen);
+    p->tx_last->pos += cpylen;
+
+    payload += cpylen;
+    size -= cpylen;
   }
+
+  if (!p->sk->tbuf && !ev_active(p->tx_ev))
+    ev_schedule(p->tx_ev);
+}
+
+static void
+bmp_tx_buffer_free(struct bmp_proto *p, struct bmp_tx_buffer *btb)
+{
+  if (btb == p->tx_last)
+  {
+    p->tx_last = NULL;
+    ASSERT_DIE(!p->tx_pending_count);
+  }
+
+  free_page(btb);
 }
 
 static void
@@ -300,40 +391,33 @@ bmp_fire_tx(void *p_)
   if (!p->started)
     return;
 
-  IF_COND_TRUE_PRINT_ERR_MSG_AND_RETURN_OPT_VAL(
-    EMPTY_LIST(p->tx_queue),
-    "Called BMP TX event handler when there is not any data to send"
-  );
-
-  size_t cnt = 0; // Counts max packets which we want to send per TX slot
-  struct bmp_data_node *tx_data;
-  struct bmp_data_node *tx_data_next;
-  WALK_LIST_DELSAFE(tx_data, tx_data_next, p->tx_queue)
+  int cnt = 0;
+  for (struct bmp_tx_buffer *btb; btb = p->tx_pending; )
   {
-    if (tx_data->data_size > p->sk->tbsize)
-    {
-      sk_set_tbsize(p->sk, tx_data->data_size);
-    }
+    ASSERT_DIE(!p->sk->tbuf);
 
-    size_t data_size = tx_data->data_size;
-    memcpy(p->sk->tbuf, tx_data->data, data_size);
-    mb_free(tx_data->data);
-    rem_node((node *) tx_data);
-    mb_free(tx_data);
+    p->sk->tbuf = btb->data;
+    u64 sz = btb->pos - btb->data;
 
-    if (sk_send(p->sk, data_size) <= 0)
+    p->tx_sent += sz;
+    p->tx_sent_total += sz;
+
+    if (p->tx_pending == p->tx_last)
+      p->tx_last = NULL;
+
+    p->tx_pending = btb->next;
+    p->tx_pending_count--;
+
+    if (sk_send(p->sk, sz) <= 0)
       return;
 
-    // BMP packets should be treat with lowest priority when scheduling sending
-    // packets to target. That's why we want to send max. 32 packets per event
-    // call
-    if (++cnt > 32)
+    p->sk->tbuf = NULL;
+    bmp_tx_buffer_free(p, btb);
+
+    if (cnt++ > 1024)
     {
       if (!ev_active(p->tx_ev))
-      {
-        ev_schedule(p->tx_ev);
-      }
-
+	ev_schedule(p->tx_ev);
       return;
     }
   }
@@ -342,6 +426,13 @@ bmp_fire_tx(void *p_)
 static void
 bmp_tx(struct birdsock *sk)
 {
+  struct bmp_proto *p = sk->data;
+
+  struct bmp_tx_buffer *btb = SKIP_BACK(struct bmp_tx_buffer, data, sk->tbuf);
+  bmp_tx_buffer_free(p, btb);
+
+  sk->tbuf = NULL;
+
   bmp_fire_tx(sk->data);
 }
 
@@ -420,21 +511,31 @@ bmp_per_peer_hdr_serialize(buffer *stream, const bool is_global_instance_peer,
 }
 
 /* [4.6] Route Monitoring */
-static void
-bmp_route_monitor_msg_serialize(buffer *stream, const bool is_peer_global,
+static byte *
+bmp_route_monitor_msg_serialize(struct bmp_proto *p, const bool is_peer_global,
   const bool table_in_post_policy, const u32 peer_as, const u32 peer_bgp_id,
-  const bool as4_support, const ip_addr remote_addr, const byte *update_msg,
+  const bool as4_support, const ip_addr remote_addr, byte *update_msg,
   const size_t update_msg_size, btime timestamp)
 {
+  ASSERT_DIE(update_msg < &p->msgbuf[sizeof p->msgbuf]);
+
+  buffer stream;
+  STACK_BUFFER_INIT(stream, BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE);
+
   const size_t data_size = BMP_PER_PEER_HDR_SIZE + update_msg_size;
   u32 ts_sec = timestamp TO_S;
   u32 ts_usec = timestamp - (ts_sec S);
 
-  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
-  bmp_common_hdr_serialize(stream, BMP_ROUTE_MONITOR, data_size);
-  bmp_per_peer_hdr_serialize(stream, is_peer_global, table_in_post_policy,
+  bmp_common_hdr_serialize(&stream, BMP_ROUTE_MONITOR, data_size);
+  bmp_per_peer_hdr_serialize(&stream, is_peer_global, table_in_post_policy,
     as4_support, remote_addr, peer_as, peer_bgp_id, ts_sec, ts_usec);
-  bmp_put_data(stream, update_msg, update_msg_size);
+
+  size_t hdr_sz = stream.pos - stream.start;
+  ASSERT_DIE(update_msg >= &p->msgbuf[hdr_sz]);
+
+  byte *begin = &update_msg[-hdr_sz];
+  memcpy(begin, stream.start, hdr_sz);
+  return begin;
 }
 
 static void
@@ -453,6 +554,7 @@ bmp_peer_up_notif_msg_serialize(buffer *stream, const bool is_peer_global,
   bmp_per_peer_hdr_serialize(stream, is_peer_global,
     false /* TODO: Hardcoded pre-policy Adj-RIB-In */, as4_support, remote_addr,
     peer_as, peer_bgp_id, 0, 0); // 0, 0 - No timestamp provided
+
   bmp_put_ipa(stream, local_addr);
   bmp_put_u16(stream, local_port);
   bmp_put_u16(stream, remote_port);
@@ -465,15 +567,37 @@ bmp_peer_up_notif_msg_serialize(buffer *stream, const bool is_peer_global,
 static void
 bmp_peer_down_notif_msg_serialize(buffer *stream, const bool is_peer_global,
   const u32 peer_as, const u32 peer_bgp_id, const bool as4_support,
-  const ip_addr remote_addr, const byte *data, const size_t data_size)
+  const ip_addr remote_addr, const struct bmp_peer_down_info *info)
 {
-  const size_t payload_size = BMP_PER_PEER_HDR_SIZE + data_size;
-  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + payload_size);
-  bmp_common_hdr_serialize(stream, BMP_PEER_DOWN_NOTIF, payload_size);
+  const size_t data_size = BMP_PER_PEER_HDR_SIZE + 1 +
+    (((info->reason == BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION) ||
+      (info->reason == BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION)) ? (BGP_HEADER_LENGTH + 2 + info->length) :
+     (info->reason == BMP_PEER_DOWN_REASON_LOCAL_NO_NOTIFICATION) ? 2 : 0);
+
+  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
+  bmp_common_hdr_serialize(stream, BMP_PEER_DOWN_NOTIF, data_size);
   bmp_per_peer_hdr_serialize(stream, is_peer_global,
     false /* TODO: Hardcoded pre-policy adj RIB IN */,  as4_support, remote_addr,
     peer_as, peer_bgp_id, 0, 0); // 0, 0 - No timestamp provided
-  bmp_put_data(stream, data, data_size);
+
+  bmp_put_u8(stream, info->reason);
+
+  switch (info->reason)
+  {
+  case BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION:
+  case BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION:;
+    uint bgp_msg_length = BGP_HEADER_LENGTH + 2 + info->length;
+    bmp_buffer_need(stream, bgp_msg_length);
+    bmp_put_bgp_hdr(stream, bgp_msg_length, PKT_NOTIFICATION);
+    bmp_put_u8(stream, info->err_code);
+    bmp_put_u8(stream, info->err_subcode);
+    bmp_put_data(stream, info->data, info->length);
+    break;
+
+  case BMP_PEER_DOWN_REASON_LOCAL_NO_NOTIFICATION:
+    bmp_put_u16(stream, info->fsm_code);
+    break;
+  }
 }
 
 
@@ -765,99 +889,63 @@ bmp_send_peer_up_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
   ASSERT(p->started);
 
   const struct birdsock *sk = bmp_get_birdsock_ext(bgp);
-  IF_PTR_IS_NULL_PRINT_ERR_MSG_AND_RETURN_OPT_VAL(
-    sk,
-    "[BMP] No BGP socket"
-  );
+  if (!sk)
+  {
+    log(L_WARN "%s: No BGP socket", p->p.name);
+    return;
+  }
 
   const bool is_global_instance_peer = bmp_is_peer_global_instance(bgp);
-  buffer payload = bmp_buffer_alloc(p->buffer_mpool, DEFAULT_MEM_BLOCK_SIZE);
+  buffer payload = bmp_default_buffer(p);
   bmp_peer_up_notif_msg_serialize(&payload, is_global_instance_peer,
     bgp->remote_as, bgp->remote_id, 1,
     sk->saddr, sk->daddr, sk->sport, sk->dport, tx_data, tx_data_size,
     rx_data, rx_data_size);
-  bmp_schedule_tx_packet(p, bmp_buffer_data(&payload), bmp_buffer_pos(&payload));
-  bmp_buffer_free(&payload);
+  bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
 }
 
 static void
-bmp_route_monitor_put_update(struct bmp_proto *p, struct bmp_stream *bs, const byte *data, size_t length, btime timestamp)
+bmp_route_monitor_put_update(struct bmp_proto *p, struct bmp_stream *bs, byte *data, size_t length, btime timestamp)
 {
-  struct bmp_data_node *upd_msg = mb_alloc(p->update_msg_mem_pool,
-                               sizeof (struct bmp_data_node));
-  upd_msg->data = mb_alloc(p->update_msg_mem_pool, length);
-  memcpy(upd_msg->data, data, length);
-  upd_msg->data_size = length;
-  add_tail(&p->update_msg_queue, &upd_msg->n);
-
-  /* Save some metadata */
   struct bgp_proto *bgp = bs->bgp;
-  upd_msg->remote_as = bgp->remote_as;
-  upd_msg->remote_id = bgp->remote_id;
-  upd_msg->remote_ip = bgp->remote_ip;
-  upd_msg->timestamp = timestamp;
-  upd_msg->global_peer = bmp_is_peer_global_instance(bgp);
-  upd_msg->policy = bmp_stream_policy(bs);
+  const byte *start = bmp_route_monitor_msg_serialize(p,
+      bmp_is_peer_global_instance(bgp),
+      bmp_stream_policy(bs),
+      bgp->remote_as,
+      bgp->remote_id,
+      true,
+      bgp->remote_ip,
+      data,
+      length,
+      timestamp
+      );
 
-  /* Kick the commit */
-  if (!ev_active(p->update_ev))
-    ev_schedule(p->update_ev);
+  bmp_schedule_tx_packet(p, start, (data - start) + length);
 }
 
 static void
 bmp_route_monitor_notify(struct bmp_proto *p, struct bmp_stream *bs,
 			 const net_addr *n, const struct rte *new, const struct rte_src *src)
 {
-  byte buf[BGP_MAX_EXT_MSG_LENGTH];
-  byte *end = bgp_bmp_encode_rte(bs->sender, buf, n, new, src);
+  byte *bufend = &p->msgbuf[sizeof p->msgbuf];
+  byte *begin = bufend - BGP_MAX_EXT_MSG_LENGTH;
+  byte *end = bgp_bmp_encode_rte(bs->sender, begin, bufend, n, new, src);
 
   btime delta_t = new ? current_time() - new->lastmod : 0;
   btime timestamp = current_real_time() - delta_t;
 
   if (end)
-    bmp_route_monitor_put_update(p, bs, buf, end - buf, timestamp);
+    bmp_route_monitor_put_update(p, bs, begin, end - begin, timestamp);
   else
     log(L_WARN "%s: Cannot encode update for %N", p->p.name, n);
 }
 
-static void
-bmp_route_monitor_commit(void *p_)
-{
-  struct bmp_proto *p = p_;
-
-  if (!p->started)
-    return;
-
-  buffer payload
-    = bmp_buffer_alloc(p->buffer_mpool, DEFAULT_MEM_BLOCK_SIZE);
-
-  struct bmp_data_node *data, *data_next;
-  WALK_LIST_DELSAFE(data, data_next, p->update_msg_queue)
-  {
-    bmp_route_monitor_msg_serialize(&payload,
-      data->global_peer, data->policy,
-      data->remote_as, data->remote_id, true,
-      data->remote_ip, data->data, data->data_size,
-      data->timestamp);
-
-    bmp_schedule_tx_packet(p, bmp_buffer_data(&payload), bmp_buffer_pos(&payload));
-
-    bmp_buffer_flush(&payload);
-
-    mb_free(data->data);
-    rem_node(&data->n);
-    mb_free(data);
-  }
-
-  bmp_buffer_free(&payload);
-}
-
 static void
 bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_stream *bs)
 {
   TRACE(D_PACKETS, "Sending END-OF-RIB for %s.%s", bs->bgp->p.name, bs->sender->c.name);
 
-  byte rx_end_payload[DEFAULT_MEM_BLOCK_SIZE];
+  byte *rx_end_payload = p->msgbuf + BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE;
   byte *pos = bgp_create_end_mark_(bs->sender, rx_end_payload + BGP_HEADER_LENGTH);
   memset(rx_end_payload + BGP_MSG_HDR_MARKER_POS, 0xff,
 	 BGP_MSG_HDR_MARKER_SIZE); // BGP UPDATE MSG marker
@@ -869,21 +957,18 @@ bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_stream *bs)
 
 static void
 bmp_send_peer_down_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
-  const byte *data, const size_t data_size)
+			     const struct bmp_peer_down_info *info)
 {
   ASSERT(p->started);
 
   const struct bgp_caps *remote_caps = bmp_get_bgp_remote_caps_ext(bgp);
   bool is_global_instance_peer = bmp_is_peer_global_instance(bgp);
-  buffer payload
-    = bmp_buffer_alloc(p->buffer_mpool, DEFAULT_MEM_BLOCK_SIZE);
+  buffer payload = bmp_default_buffer(p);
   bmp_peer_down_notif_msg_serialize(&payload, is_global_instance_peer,
     bgp->remote_as, bgp->remote_id,
     remote_caps ? remote_caps->as4_support : bgp->as4_session,
-    bgp->remote_ip, data, data_size);
-  bmp_schedule_tx_packet(p, bmp_buffer_data(&payload), bmp_buffer_pos(&payload));
-
-  bmp_buffer_free(&payload);
+    bgp->remote_ip, info);
+  bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
 }
 
 static void
@@ -899,48 +984,32 @@ bmp_peer_down_(struct bmp_proto *p, const struct bgp_proto *bgp,
 
   TRACE(D_STATES, "Peer down for %s", bgp->p.name);
 
-  uint bmp_code = 0;
-  uint fsm_code = 0;
+  struct bmp_peer_down_info info = {
+    .err_code = err_code,
+    .err_subcode = err_subcode,
+    .data = data,
+    .length = length,
+  };
 
   switch (err_class)
   {
   case BE_BGP_RX:
-    bmp_code = BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION;
+    info.reason = BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION;
     break;
 
   case BE_BGP_TX:
   case BE_AUTO_DOWN:
   case BE_MAN_DOWN:
-    bmp_code = BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION;
+    info.reason = BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION;
     break;
 
   default:
-    bmp_code = BMP_PEER_DOWN_REASON_REMOTE_NO_NOTIFICATION;
-    length = 0;
+    info.reason = BMP_PEER_DOWN_REASON_REMOTE_NO_NOTIFICATION;
+    info.length = 0;
     break;
   }
 
-  buffer payload = bmp_buffer_alloc(p->buffer_mpool, 1 + BGP_HEADER_LENGTH + 2 + length);
-  bmp_put_u8(&payload, bmp_code);
-
-  switch (bmp_code)
-  {
-  case BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION:
-  case BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION:
-    bmp_put_bgp_hdr(&payload, BGP_HEADER_LENGTH + 2 + length, PKT_NOTIFICATION);
-    bmp_put_u8(&payload, err_code);
-    bmp_put_u8(&payload, err_subcode);
-    bmp_put_data(&payload, data, length);
-    break;
-
-  case BMP_PEER_DOWN_REASON_LOCAL_NO_NOTIFICATION:
-    bmp_put_u16(&payload, fsm_code);
-    break;
-  }
-
-  bmp_send_peer_down_notif_msg(p, bgp, bmp_buffer_data(&payload), bmp_buffer_pos(&payload));
-
-  bmp_buffer_free(&payload);
+  bmp_send_peer_down_notif_msg(p, bgp, &info);
 
   bmp_remove_peer(p, bp);
 }
@@ -962,18 +1031,21 @@ bmp_send_termination_msg(struct bmp_proto *p,
                                      + BMP_TERM_INFO_LEN_FIELD_SIZE
                                      + BMP_TERM_REASON_CODE_SIZE;
   const size_t term_msg_size = BMP_COMMON_HDR_SIZE + term_msg_hdr_size;
-  buffer stream = bmp_buffer_alloc(p->buffer_mpool, term_msg_size);
+  buffer stream = bmp_default_buffer(p);
+  bmp_buffer_need(&stream, term_msg_size);
+
   bmp_common_hdr_serialize(&stream, BMP_TERM_MSG, term_msg_hdr_size);
   bmp_put_u16(&stream, BMP_TERM_INFO_REASON);
   bmp_put_u16(&stream, BMP_TERM_REASON_CODE_SIZE); // 2-byte code indication the reason
   bmp_put_u16(&stream, reason);
-  memcpy(p->sk->tbuf, bmp_buffer_data(&stream), bmp_buffer_pos(&stream));
-  IF_COND_TRUE_PRINT_ERR_MSG_AND_RETURN_OPT_VAL(
-    sk_send(p->sk, bmp_buffer_pos(&stream)) < 0,
-    "Failed to send BMP termination message"
-    );
 
-  bmp_buffer_free(&stream);
+  if (p->sk->tbuf)
+    bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data, p->sk->tbuf));
+
+  p->sk->tbuf = stream.start;
+  if (sk_send(p->sk, stream.pos - stream.start) < 0)
+    log(L_WARN "%s: Cannot send BMP termination message", p->p.name);
+  p->sk->tbuf = NULL;
 }
 
 int
@@ -1062,10 +1134,9 @@ bmp_startup(struct bmp_proto *p)
   proto_notify_state(&p->p, PS_UP);
 
   /* Send initiation message */
-  buffer payload = bmp_buffer_alloc(p->buffer_mpool, DEFAULT_MEM_BLOCK_SIZE);
+  buffer payload = bmp_default_buffer(p);
   bmp_init_msg_serialize(&payload, p->sys_descr, p->sys_name);
-  bmp_schedule_tx_packet(p, bmp_buffer_data(&payload), bmp_buffer_pos(&payload));
-  bmp_buffer_free(&payload);
+  bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
 
   /* Send Peer Up messages */
   struct proto *peer;
@@ -1086,6 +1157,7 @@ bmp_down(struct bmp_proto *p)
 {
   ASSERT(p->started);
   p->started = false;
+  p->tx_sent = 0;
 
   TRACE(D_EVENTS, "BMP session closed");
 
@@ -1119,7 +1191,6 @@ bmp_connect(struct bmp_proto *p)
   sk->dport = p->station_port;
   sk->ttl = IP4_MAX_TTL;
   sk->tos = IP_PREC_INTERNET_CONTROL;
-  sk->tbsize = BGP_TX_BUFFER_EXT_SIZE;
   sk->tx_hook = bmp_connected;
   sk->err_hook = bmp_sock_err;
 
@@ -1173,6 +1244,26 @@ bmp_sock_err(sock *sk, int err)
   proto_notify_state(&p->p, PS_START);
 }
 
+static void
+bmp_tx_overflow(void *_p)
+{
+  struct bmp_proto *p = _p;
+  if (p->tx_pending_count < p->tx_pending_limit)
+    return;
+
+  p->sock_err = 0;
+
+  log(L_ERR "%s: Connection stalled", p->p.name);
+
+  if (p->started)
+    bmp_down(p);
+
+  bmp_close_socket(p);
+  tm_start(p->connect_retry_timer, CONNECT_RETRY_TIME);
+
+  proto_notify_state(&p->p, PS_START);
+}
+
 /* BMP connect timeout event - switch from Idle/Connect state to Connect state */
 static void
 bmp_connection_retry(timer *t)
@@ -1189,6 +1280,24 @@ bmp_connection_retry(timer *t)
 static void
 bmp_close_socket(struct bmp_proto *p)
 {
+  if (p->sk && p->sk->tbuf)
+    bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data, p->sk->tbuf));
+
+  struct bmp_tx_buffer *btb = p->tx_pending;
+  while (btb)
+  {
+    p->tx_pending_count--;
+
+    struct bmp_tx_buffer *next = btb->next;
+    bmp_tx_buffer_free(p, btb);
+    btb = next;
+  }
+
+  p->tx_pending = NULL;
+
+  ASSERT_DIE(!p->tx_last);
+  ASSERT_DIE(!p->tx_pending_count);
+
   rfree(p->sk);
   p->sk = NULL;
 }
@@ -1230,6 +1339,7 @@ bmp_init(struct proto_config *CF)
   strcpy(p->sys_name, cf->sys_name);
   p->monitoring_rib.in_pre_policy = cf->monitoring_rib_in_pre_policy;
   p->monitoring_rib.in_post_policy = cf->monitoring_rib_in_post_policy;
+  p->tx_pending_limit = cf->tx_pending_limit;
 
   return P;
 }
@@ -1243,20 +1353,20 @@ bmp_start(struct proto *P)
 {
   struct bmp_proto *p = (void *) P;
 
-  p->buffer_mpool = rp_new(P->pool, "BMP Buffer");
-  p->tx_mem_pool = rp_new(P->pool, "BMP Tx");
-  p->update_msg_mem_pool = rp_new(P->pool, "BMP Update");
   p->tx_ev = ev_new_init(p->p.pool, bmp_fire_tx, p);
-  p->update_ev = ev_new_init(p->p.pool, bmp_route_monitor_commit, p);
+  p->tx_pending = NULL;
+  p->tx_pending_count = 0;
+  p->tx_overflow_event = ev_new_init(p->p.pool, bmp_tx_overflow, p);
   p->connect_retry_timer = tm_new_init(p->p.pool, bmp_connection_retry, p, 0, 0);
   p->sk = NULL;
 
+  resource *r = ralloc(P->pool, &bmp_tx_resource_class);
+  SKIP_BACK(struct bmp_tx_resource, r, r)->p = p;
+
   HASH_INIT(p->peer_map, P->pool, 4);
   HASH_INIT(p->stream_map, P->pool, 4);
   HASH_INIT(p->table_map, P->pool, 4);
 
-  init_list(&p->tx_queue);
-  init_list(&p->update_msg_queue);
   p->started = false;
   p->sock_err = 0;
   add_tail(&bmp_proto_list, &p->bmp_node);
@@ -1275,6 +1385,7 @@ bmp_shutdown(struct proto *P)
   {
     bmp_send_termination_msg(p, BMP_TERM_REASON_ADM);
     bmp_down(p);
+    bmp_close_socket(p);
   }
 
   p->sock_err = 0;
@@ -1305,6 +1416,9 @@ bmp_reconfigure(struct proto *P, struct proto_config *CF)
   /* We must update our copy of configuration ptr */
   p->cf = new;
 
+  /* Reconfigure tx buffer size limits */
+  p->tx_pending_limit = new->tx_pending_limit;
+
   return 1;
 }
 
@@ -1341,6 +1455,13 @@ bmp_show_proto_info(struct proto *P)
 
     if (p->sock_err)
       cli_msg(-1006, "  %-19s %M", "Last error:", p->sock_err);
+
+    cli_msg(-1006, "  %-19s % 9sB (limit %sB)", "Pending TX:",
+	fmt_order(p->tx_pending_count * (u64) page_size, 1, 10000),
+	fmt_order(p->tx_pending_limit * (u64) page_size, 1, 10000));
+
+    cli_msg(-1006, "  %-19s % 9sB", "Session TX:", fmt_order(p->tx_sent, 1, 10000));
+    cli_msg(-1006, "  %-19s % 9sB", "Total TX:", fmt_order(p->tx_sent_total, 1, 10000));
   }
 }
 
diff --git a/proto/bmp/bmp.h b/proto/bmp/bmp.h
index 45844836..52f72122 100644
--- a/proto/bmp/bmp.h
+++ b/proto/bmp/bmp.h
@@ -16,11 +16,22 @@
 #include "lib/event.h"
 #include "lib/hash.h"
 #include "lib/socket.h"
+#include "proto/bgp/bgp.h"
 #include "proto/bmp/map.h"
 
 // Max length of MIB-II description object
 #define MIB_II_STR_LEN 255
 
+// Total size of Common Header
+#define BMP_COMMON_HDR_SIZE 6
+
+/* BMP Per-Peer Header [RFC 7854 - Section 4.2] */
+// Total size of Per-Peer Header
+#define BMP_PER_PEER_HDR_SIZE 42
+
+// Maximum length of BMP message altogether
+#define BMP_MSGBUF_LEN (BGP_MAX_EXT_MSG_LENGTH + BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE + 1)
+
 // The following fields of this structure controls whether there will be put
 // specific routes into Route Monitoring message and send to BMP collector
 struct monitoring_rib {
@@ -38,6 +49,7 @@ struct bmp_config {
   u16 station_port;                   // Monitoring station TCP port
   bool monitoring_rib_in_pre_policy;  // Route monitoring pre-policy Adj-Rib-In
   bool monitoring_rib_in_post_policy;  // Route monitoring post-policy Adj-Rib-In
+  uint tx_pending_limit;	      // Maximum on pending TX buffer count
 };
 
 /* Forward declarations */
@@ -64,14 +76,17 @@ struct bmp_proto {
   struct monitoring_rib monitoring_rib;
   // Below fields are for internal use
   // struct bmp_peer_map bgp_peers;   // Stores 'bgp_proto' structure per BGP peer
-  pool *buffer_mpool;              // Memory pool used for BMP buffer allocations
-  pool *tx_mem_pool;               // Memory pool used for packet allocations designated to BMP collector
-  pool *update_msg_mem_pool;       // Memory pool used for BPG UPDATE MSG allocations
-  list tx_queue;                   // Stores queued packets going to be sent
+  struct bmp_tx_buffer *tx_pending;// This buffer waits for socket to flush
+  struct bmp_tx_buffer *tx_last;   // This buffer is the last to flush 
+  uint tx_pending_count;	   // How many buffers waiting for flush
+  uint tx_pending_limit;	   // Maximum on buffer count
+  u64 tx_sent;			   // Amount of data sent
+  u64 tx_sent_total;		   // Amount of data sent accumulated over reconnections
+  event *tx_overflow_event;	   // Too many buffers waiting for flush
   timer *connect_retry_timer;      // Timer for retrying connection to the BMP collector
-  list update_msg_queue;           // Stores all composed BGP UPDATE MSGs
   bool started;                    // Flag that stores running status of BMP instance
   int sock_err;                    // Last socket error code
+  byte msgbuf[BMP_MSGBUF_LEN];     // Buffer for preparing the messages before sending them out
 };
 
 struct bmp_peer {
diff --git a/proto/bmp/buffer.c b/proto/bmp/buffer.c
deleted file mode 100644
index be9dd698..00000000
--- a/proto/bmp/buffer.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- *	BIRD -- The BGP Monitoring Protocol (BMP)
- *
- *	(c) 2020 Akamai Technologies, Inc. (Pawel Maslanka, pmaslank@akamai.com)
- *
- *	Can be freely distributed and used under the terms of the GNU GPL.
- */
-
-#include "proto/bmp/buffer.h"
-
-buffer
-bmp_buffer_alloc(pool *ppool, const size_t n)
-{
-  buffer buf;
-  buf.start = mb_alloc(ppool, n);
-  buf.pos = buf.start;
-  buf.end = buf.start + n;
-  return buf;
-}
-
-void
-bmp_buffer_free(buffer *buf)
-{
-  mb_free(buf->start);
-  buf->start = buf->pos = buf->end = NULL;
-}
-
-/**
- * @brief bmp_buffer_grow
- * @param buf - buffer to grow
- * @param n   - required amount of available space
- * Resize buffer in a way that there is at least @n bytes of available space.
- */
-static void
-bmp_buffer_grow(buffer *buf, const size_t n)
-{
-  size_t pos = bmp_buffer_pos(buf);
-  size_t size = bmp_buffer_size(buf);
-  size_t req = pos + n;
-
-  while (size < req)
-    size = size * 3 / 2;
-
-  buf->start = mb_realloc(buf->start, size);
-  buf->pos = buf->start + pos;
-  buf->end = buf->start + size;
-}
-
-void
-bmp_buffer_need(buffer *buf, const size_t n)
-{
-  if (bmp_buffer_avail(buf) < n)
-    bmp_buffer_grow(buf, n);
-}
-
-void
-bmp_put_data(buffer *buf, const void *src, const size_t n)
-{
-  if (!n)
-    return;
-
-  bmp_buffer_need(buf, n);
-  memcpy(buf->pos, src, n);
-  buf->pos += n;
-}
diff --git a/proto/bmp/buffer.h b/proto/bmp/buffer.h
deleted file mode 100644
index f752cf5d..00000000
--- a/proto/bmp/buffer.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- *	BIRD -- The BGP Monitoring Protocol (BMP)
- *
- *	(c) 2020 Akamai Technologies, Inc. (Pawel Maslanka, pmaslank@akamai.com)
- *
- *	Can be freely distributed and used under the terms of the GNU GPL.
- */
-
-#ifndef _BIRD_BMP_BUFFER_H_
-#define _BIRD_BMP_BUFFER_H_
-
-#include "proto/bmp/bmp.h"
-
-#include <stdlib.h>
-
-#include "lib/resource.h"
-
-buffer
-bmp_buffer_alloc(pool *ppool, const size_t n);
-
-void
-bmp_buffer_free(buffer *buf);
-
-static inline void
-bmp_buffer_flush(buffer *buf)
-{
-  buf->pos = buf->start;
-}
-
-static inline size_t
-bmp_buffer_size(const buffer *buf)
-{
-  return buf->end - buf->start;
-}
-
-static inline size_t
-bmp_buffer_avail(const buffer *buf)
-{
-  return buf->end - buf->pos;
-}
-
-static inline size_t
-bmp_buffer_pos(const buffer *buf)
-{
-  return buf->pos - buf->start;
-}
-
-static inline byte *
-bmp_buffer_data(const buffer *buf)
-{
-  return buf->start;
-}
-
-void
-bmp_buffer_need(buffer *buf, const size_t n);
-
-// Idea for following macros has been taken from |proto/mrt/mrt.c|
-#define BMP_DEFINE_PUT_FUNC(S, T)                               \
-  static inline void                                            \
-  bmp_put_##S(buffer *b, const T x)                             \
-  {                                                             \
-    bmp_buffer_need(b, sizeof(T));                              \
-    put_##S(b->pos, x);                                    \
-    b->pos += sizeof(T);                                   \
-  }
-
-BMP_DEFINE_PUT_FUNC(u8, u8)
-BMP_DEFINE_PUT_FUNC(u16, u16)
-BMP_DEFINE_PUT_FUNC(u32, u32)
-BMP_DEFINE_PUT_FUNC(u64, u64)
-BMP_DEFINE_PUT_FUNC(ip4, ip4_addr)
-BMP_DEFINE_PUT_FUNC(ip6, ip6_addr)
-
-void
-bmp_put_data(buffer *buf, const void *src, const size_t n);
-
-#endif /* _BIRD_BMP_BUFFER_H_ */
diff --git a/proto/bmp/config.Y b/proto/bmp/config.Y
index acb0c4d9..04540b26 100644
--- a/proto/bmp/config.Y
+++ b/proto/bmp/config.Y
@@ -27,6 +27,7 @@ bmp_proto_start: proto_start BMP {
      this_proto = proto_config_new(&proto_bmp, $1);
      BMP_CFG->sys_descr = "Not defined";
      BMP_CFG->sys_name = "Not defined";
+     BMP_CFG->tx_pending_limit = (1 << 30) / page_size;
    }
  ;
 
@@ -71,6 +72,10 @@ bmp_proto:
  | bmp_proto MONITORING RIB IN POST_POLICY bool ';' {
      BMP_CFG->monitoring_rib_in_post_policy = $6;
    }
+ | bmp_proto TX BUFFER LIMIT expr ';' {
+     BMP_CFG->tx_pending_limit = $5 * (u64) (1 << 20) / page_size;
+     if ($5 < 1) cf_error("BMP TX buffer limit must be at least 1 megabyte");
+   }
  ;
 
 CF_CODE

From 5205ff97448cc34cf7334e90172c28eb48f227f2 Mon Sep 17 00:00:00 2001
From: Ondrej Zajicek <santiago@crfreenet.org>
Date: Sun, 1 Dec 2024 20:56:53 +0100
Subject: [PATCH 04/15] BMP: Refactor per-peer header arguments

Put all arguments for per-peer header into one structure instead of
passing them around as a lump of arguments. Also simplify code for
peer flags.
---
 proto/bmp/bmp.c | 170 +++++++++++++++++++++---------------------------
 1 file changed, 74 insertions(+), 96 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 6dcf1562..1af101ae 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -103,32 +103,20 @@ enum bmp_peer_type {
   BMP_PEER_TYPE_LOCAL_INSTANCE = 2
 };
 
-#define BMP_PEER_HDR_FLAG_V_SHIFT 7
-enum bmp_peer_flag_v_t {
-  // The Peer address is an IPv4 address
-  BMP_PEER_HDR_FLAG_V_IP4 = (0 << BMP_PEER_HDR_FLAG_V_SHIFT),
-  // The Peer address is an IPv6 address
-  BMP_PEER_HDR_FLAG_V_IP6 = (1 << BMP_PEER_HDR_FLAG_V_SHIFT)
+#define BMP_PEER_FLAG_V_IPV6		(1 << 7)
+#define BMP_PEER_FLAG_L_POLICY		(1 << 6)
+#define BMP_PEER_FLAG_A_NO_AS4		(1 << 5)
+
+struct bmp_peer_hdr_info {
+  ip_addr address;
+  u32 as;
+  u32 id;
+  bool global;
+  bool policy;
+  bool no_as4;
+  btime timestamp;
 };
 
-#define BMP_PEER_HDR_FLAG_L_SHIFT 6
-enum bmp_peer_flag_l {
-  BMP_PEER_HDR_FLAG_L_PRE_POLICY_ADJ_RIB_IN = (0 << BMP_PEER_HDR_FLAG_L_SHIFT),
-  BMP_PEER_HDR_FLAG_L_POST_POLICY_ADJ_RIB_IN = (1 << BMP_PEER_HDR_FLAG_L_SHIFT)
-};
-
-#define BMP_PEER_HDR_FLAG_A_SHIFT 5
-enum bmp_peer_flag_a {
-  // The 4-byte AS_PATH format
-  BMP_PEER_HDR_FLAG_A_AS_PATH_4B = (0 << BMP_PEER_HDR_FLAG_A_SHIFT),
-  // The legacy 2-byte AS_PATH format
-  BMP_PEER_HDR_FLAG_A_AS_PATH_2B = (1 << BMP_PEER_HDR_FLAG_A_SHIFT)
-};
-
-#define BMP_PEER_HDR_FLAGS_INIT(flags) \
-  (flags) = 0
-#define BMP_PEER_HDR_FLAGS_SET(flags, bit_mask) \
-  (flags) |= (bit_mask)
 
 /* BMP Information TLV header [RFC 7854 - Section 4.4] */
 // Total size of Type and Length fields of Information TLV Header without
@@ -475,60 +463,44 @@ bmp_put_bgp_hdr(buffer *stream, const u8 msg_type, const u16 msg_length)
  * @ts_usec: the time in microseconds when the encapsulated routes were received
  */
 static void
-bmp_per_peer_hdr_serialize(buffer *stream, const bool is_global_instance_peer,
-  const bool is_post_policy, const bool is_as_path_4bytes,
-  const ip_addr peer_addr, const u32 peer_as, const u32 peer_bgp_id,
-  const u32 ts_sec, const u32 ts_usec)
+bmp_per_peer_hdr_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer)
 {
-  // TODO: ATM we don't support BMP_PEER_TYPE_RD_INSTANCE
-  const enum bmp_peer_type peer_type = is_global_instance_peer
-                                      ? BMP_PEER_TYPE_GLOBAL_INSTANCE
-                                      : BMP_PEER_TYPE_LOCAL_INSTANCE;
-  const u8 peer_flag_v = ipa_is_ip4(peer_addr)
-                           ? BMP_PEER_HDR_FLAG_V_IP4
-                           : BMP_PEER_HDR_FLAG_V_IP6;
-  const u8 peer_flag_l = is_post_policy
-                           ? BMP_PEER_HDR_FLAG_L_POST_POLICY_ADJ_RIB_IN
-                           : BMP_PEER_HDR_FLAG_L_PRE_POLICY_ADJ_RIB_IN;
-  const u8 peer_flag_a = is_as_path_4bytes
-                           ? BMP_PEER_HDR_FLAG_A_AS_PATH_4B
-                           : BMP_PEER_HDR_FLAG_A_AS_PATH_2B;
-  u8 peer_flags;
-  BMP_PEER_HDR_FLAGS_INIT(peer_flags);
-  BMP_PEER_HDR_FLAGS_SET(peer_flags, peer_flag_v);
-  BMP_PEER_HDR_FLAGS_SET(peer_flags, peer_flag_l);
-  BMP_PEER_HDR_FLAGS_SET(peer_flags, peer_flag_a);
+  // TODO: ATM we do not support BMP_PEER_TYPE_RD_INSTANCE
+  u8 peer_type = peer->global ?
+    BMP_PEER_TYPE_GLOBAL_INSTANCE :
+    BMP_PEER_TYPE_LOCAL_INSTANCE;
+
+  u8 peer_flags =
+    (ipa_is_ip6(peer->address) ? BMP_PEER_FLAG_V_IPV6 : 0) |
+    (peer->policy ? BMP_PEER_FLAG_L_POLICY : 0) |
+    (peer->no_as4 ? BMP_PEER_FLAG_A_NO_AS4 : 0);
+
+  u32 ts_sec = peer->timestamp TO_S;
+  u32 ts_usec = peer->timestamp - (ts_sec S);
 
   bmp_put_u8(stream, peer_type);
   bmp_put_u8(stream, peer_flags);
-  // TODO: Provide appropriate peer Route Distinguisher if applicable
-  bmp_put_u64(stream, 0x00); // 0x00 - Not supported peer distinguisher
-  bmp_put_ipa(stream, peer_addr);
-  bmp_put_u32(stream, peer_as);
-  bmp_put_u32(stream, peer_bgp_id);
+  bmp_put_u64(stream, 0); // Not supported peer distinguisher
+  bmp_put_ipa(stream, peer->address);
+  bmp_put_u32(stream, peer->as);
+  bmp_put_u32(stream, peer->id);
   bmp_put_u32(stream, ts_sec);
   bmp_put_u32(stream, ts_usec);
 }
 
 /* [4.6] Route Monitoring */
 static byte *
-bmp_route_monitor_msg_serialize(struct bmp_proto *p, const bool is_peer_global,
-  const bool table_in_post_policy, const u32 peer_as, const u32 peer_bgp_id,
-  const bool as4_support, const ip_addr remote_addr, byte *update_msg,
-  const size_t update_msg_size, btime timestamp)
+bmp_route_monitor_msg_serialize(struct bmp_proto *p, const struct bmp_peer_hdr_info *peer,
+				byte *update_msg, size_t update_msg_length)
 {
   ASSERT_DIE(update_msg < &p->msgbuf[sizeof p->msgbuf]);
 
   buffer stream;
   STACK_BUFFER_INIT(stream, BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE);
 
-  const size_t data_size = BMP_PER_PEER_HDR_SIZE + update_msg_size;
-  u32 ts_sec = timestamp TO_S;
-  u32 ts_usec = timestamp - (ts_sec S);
-
+  const size_t data_size = BMP_PER_PEER_HDR_SIZE + update_msg_length;
   bmp_common_hdr_serialize(&stream, BMP_ROUTE_MONITOR, data_size);
-  bmp_per_peer_hdr_serialize(&stream, is_peer_global, table_in_post_policy,
-    as4_support, remote_addr, peer_as, peer_bgp_id, ts_sec, ts_usec);
+  bmp_per_peer_hdr_serialize(&stream, peer);
 
   size_t hdr_sz = stream.pos - stream.start;
   ASSERT_DIE(update_msg >= &p->msgbuf[hdr_sz]);
@@ -539,11 +511,10 @@ bmp_route_monitor_msg_serialize(struct bmp_proto *p, const bool is_peer_global,
 }
 
 static void
-bmp_peer_up_notif_msg_serialize(buffer *stream, const bool is_peer_global,
-  const u32 peer_as, const u32 peer_bgp_id, const bool as4_support,
-  const ip_addr local_addr, const ip_addr remote_addr, const u16 local_port,
-  const u16 remote_port, const byte *sent_msg, const size_t sent_msg_length,
-  const byte *recv_msg, const size_t recv_msg_length)
+bmp_peer_up_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer,
+				const ip_addr local_addr, const u16 local_port, const u16 remote_port,
+				const byte *sent_msg, const size_t sent_msg_length,
+				const byte *recv_msg, const size_t recv_msg_length)
 {
   const size_t data_size =
     BMP_PER_PEER_HDR_SIZE + BMP_PEER_UP_NOTIF_MSG_FIX_SIZE +
@@ -551,9 +522,7 @@ bmp_peer_up_notif_msg_serialize(buffer *stream, const bool is_peer_global,
 
   bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
   bmp_common_hdr_serialize(stream, BMP_PEER_UP_NOTIF, data_size);
-  bmp_per_peer_hdr_serialize(stream, is_peer_global,
-    false /* TODO: Hardcoded pre-policy Adj-RIB-In */, as4_support, remote_addr,
-    peer_as, peer_bgp_id, 0, 0); // 0, 0 - No timestamp provided
+  bmp_per_peer_hdr_serialize(stream, peer);
 
   bmp_put_ipa(stream, local_addr);
   bmp_put_u16(stream, local_port);
@@ -565,9 +534,8 @@ bmp_peer_up_notif_msg_serialize(buffer *stream, const bool is_peer_global,
 }
 
 static void
-bmp_peer_down_notif_msg_serialize(buffer *stream, const bool is_peer_global,
-  const u32 peer_as, const u32 peer_bgp_id, const bool as4_support,
-  const ip_addr remote_addr, const struct bmp_peer_down_info *info)
+bmp_peer_down_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer,
+				  const struct bmp_peer_down_info *info)
 {
   const size_t data_size = BMP_PER_PEER_HDR_SIZE + 1 +
     (((info->reason == BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION) ||
@@ -576,9 +544,7 @@ bmp_peer_down_notif_msg_serialize(buffer *stream, const bool is_peer_global,
 
   bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
   bmp_common_hdr_serialize(stream, BMP_PEER_DOWN_NOTIF, data_size);
-  bmp_per_peer_hdr_serialize(stream, is_peer_global,
-    false /* TODO: Hardcoded pre-policy adj RIB IN */,  as4_support, remote_addr,
-    peer_as, peer_bgp_id, 0, 0); // 0, 0 - No timestamp provided
+  bmp_per_peer_hdr_serialize(stream, peer);
 
   bmp_put_u8(stream, info->reason);
 
@@ -895,12 +861,18 @@ bmp_send_peer_up_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
     return;
   }
 
-  const bool is_global_instance_peer = bmp_is_peer_global_instance(bgp);
+  struct bmp_peer_hdr_info peer = {
+    .address = bgp->remote_ip,
+    .as = bgp->remote_as,
+    .id = bgp->remote_id,
+    .global = bmp_is_peer_global_instance(bgp),
+    .policy = false,		// Hardcoded pre-policy Adj-RIB-In
+    .timestamp = 0,		// No timestamp provided
+  };
+
   buffer payload = bmp_default_buffer(p);
-  bmp_peer_up_notif_msg_serialize(&payload, is_global_instance_peer,
-    bgp->remote_as, bgp->remote_id, 1,
-    sk->saddr, sk->daddr, sk->sport, sk->dport, tx_data, tx_data_size,
-    rx_data, rx_data_size);
+  bmp_peer_up_notif_msg_serialize(&payload, &peer, sk->saddr, sk->sport, sk->dport,
+				  tx_data, tx_data_size, rx_data, rx_data_size);
   bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
 }
 
@@ -908,18 +880,17 @@ static void
 bmp_route_monitor_put_update(struct bmp_proto *p, struct bmp_stream *bs, byte *data, size_t length, btime timestamp)
 {
   struct bgp_proto *bgp = bs->bgp;
-  const byte *start = bmp_route_monitor_msg_serialize(p,
-      bmp_is_peer_global_instance(bgp),
-      bmp_stream_policy(bs),
-      bgp->remote_as,
-      bgp->remote_id,
-      true,
-      bgp->remote_ip,
-      data,
-      length,
-      timestamp
-      );
 
+  struct bmp_peer_hdr_info peer = {
+    .address = bgp->remote_ip,
+    .as = bgp->remote_as,
+    .id = bgp->remote_id,
+    .global = bmp_is_peer_global_instance(bgp),
+    .policy = bmp_stream_policy(bs),
+    .timestamp = timestamp,
+  };
+
+  const byte *start = bmp_route_monitor_msg_serialize(p, &peer, data, length);
   bmp_schedule_tx_packet(p, start, (data - start) + length);
 }
 
@@ -962,12 +933,19 @@ bmp_send_peer_down_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
   ASSERT(p->started);
 
   const struct bgp_caps *remote_caps = bmp_get_bgp_remote_caps_ext(bgp);
-  bool is_global_instance_peer = bmp_is_peer_global_instance(bgp);
+
+  struct bmp_peer_hdr_info peer = {
+    .address = bgp->remote_ip,
+    .as = bgp->remote_as,
+    .id = bgp->remote_id,
+    .global = bmp_is_peer_global_instance(bgp),
+    .policy = false,		// Hardcoded pre-policy Adj-RIB-In
+    .no_as4 = remote_caps ? !remote_caps->as4_support : !bgp->as4_session,
+    .timestamp = 0,		// No timestamp provided
+  };
+
   buffer payload = bmp_default_buffer(p);
-  bmp_peer_down_notif_msg_serialize(&payload, is_global_instance_peer,
-    bgp->remote_as, bgp->remote_id,
-    remote_caps ? remote_caps->as4_support : bgp->as4_session,
-    bgp->remote_ip, info);
+  bmp_peer_down_notif_msg_serialize(&payload, &peer, info);
   bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
 }
 

From 73e7d3f5cede2e72eb9e77d61424a8c443672c09 Mon Sep 17 00:00:00 2001
From: Ondrej Zajicek <santiago@crfreenet.org>
Date: Sun, 1 Dec 2024 22:54:11 +0100
Subject: [PATCH 05/15] BMP: Refactor computation of message length

Fill message length after the message is created instead of computing
it beforehand. Also rename some functions to fit common pattern.
---
 proto/bmp/bmp.c | 97 ++++++++++++++++++-------------------------------
 1 file changed, 36 insertions(+), 61 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 1af101ae..dc21ac09 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -93,10 +93,6 @@ enum bmp_message_type {
   BMP_ROUTE_MIRROR_MSG = 6 // Route Mirroring Message
 };
 
-// Defines size of padding when IPv4 address is going to be put into field
-// which can accept also IPv6 address
-#define BMP_PADDING_IP4_ADDR_SIZE 12
-
 enum bmp_peer_type {
   BMP_PEER_TYPE_GLOBAL_INSTANCE = 0,
   BMP_PEER_TYPE_RD_INSTANCE = 1,
@@ -117,12 +113,6 @@ struct bmp_peer_hdr_info {
   btime timestamp;
 };
 
-
-/* BMP Information TLV header [RFC 7854 - Section 4.4] */
-// Total size of Type and Length fields of Information TLV Header without
-// variable part
-#define BMP_INFO_TLV_FIX_SIZE 4
-
 enum bmp_info_tlv_type {
   BMP_INFO_TLV_TYPE_STRING = 0,    // String
   BMP_INFO_TLV_TYPE_SYS_DESCR = 1, // SysDescr
@@ -157,14 +147,12 @@ struct bmp_peer_down_info {
 };
 
 /* BMP Termination Message [RFC 7854 - Section 4.5] */
-#define BMP_TERM_INFO_TYPE_SIZE 2
 enum bmp_term_info_type {
   BMP_TERM_INFO_STRING = 0, // The Information field contains string
   BMP_TERM_INFO_REASON = 1, // The Information field contains 2-byte reason code
 };
 
 // 2-byte code in the Information field
-#define BMP_TERM_REASON_CODE_SIZE 2
 enum bmp_term_reason {
   BMP_TERM_REASON_ADM = 0,  // Session administratively closed
   BMP_TERM_REASON_UNK = 1,  // Unspecified reason
@@ -173,12 +161,6 @@ enum bmp_term_reason {
   BMP_TERM_REASON_PERM = 4, // Session permanently administratively closed
 };
 
-// Size of Information Length field in Termination Message header
-#define BMP_TERM_INFO_LEN_FIELD_SIZE 2
-
-// Default chunk size request when memory allocation
-#define DEFAULT_MEM_BLOCK_SIZE 4096
-
 // Initial delay for connection to the BMP collector
 #define CONNECT_INIT_TIME (200 MS)
 
@@ -277,20 +259,28 @@ static struct resclass bmp_tx_resource_class = {
 };
 
 static void
-bmp_common_hdr_serialize(buffer *stream, const enum bmp_message_type type, const u32 data_size)
+bmp_put_common_hdr(buffer *stream, enum bmp_message_type type, u32 length)
 {
+  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE);
   bmp_put_u8(stream, BMP_VERSION_3);
-  bmp_put_u32(stream, BMP_COMMON_HDR_SIZE + data_size);
+  bmp_put_u32(stream, length);
   bmp_put_u8(stream, type);
 }
 
 static void
-bmp_info_tlv_hdr_serialize(buffer *stream, const enum bmp_info_tlv_type type,
-  const char *str)
+bmp_fix_common_hdr(buffer *stream)
+{
+  uint length = stream->pos - stream->start;
+  put_u32(stream->start + 1, length);
+}
+
+static void
+bmp_put_info_tlv(buffer *stream, enum bmp_info_tlv_type type, const char *str)
 {
   size_t str_len = strlen(str);
   str_len = MIN(str_len, MIB_II_STR_LEN);
 
+  bmp_buffer_need(stream, 4 + str_len);
   bmp_put_u16(stream, type);
   bmp_put_u16(stream, str_len);
   bmp_put_data(stream, str, str_len);
@@ -300,16 +290,10 @@ bmp_info_tlv_hdr_serialize(buffer *stream, const enum bmp_info_tlv_type type,
 static void
 bmp_init_msg_serialize(buffer *stream, const char *sys_descr, const char *sys_name)
 {
-  const size_t sys_descr_len = strlen(sys_descr);
-  const size_t sys_name_len = strlen(sys_name);
-  // We include MIB-II sysDescr and sysName in BMP INIT MSG so that's why
-  // allocated 2x BMP_INFO_TLV_FIX_SIZE memory pool size
-  const size_t data_size = (2 * BMP_INFO_TLV_FIX_SIZE) + sys_descr_len + sys_name_len;
-
-  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
-  bmp_common_hdr_serialize(stream, BMP_INIT_MSG, data_size);
-  bmp_info_tlv_hdr_serialize(stream, BMP_INFO_TLV_TYPE_SYS_DESCR, sys_descr);
-  bmp_info_tlv_hdr_serialize(stream, BMP_INFO_TLV_TYPE_SYS_NAME, sys_name);
+  bmp_put_common_hdr(stream, BMP_INIT_MSG, 0);
+  bmp_put_info_tlv(stream, BMP_INFO_TLV_TYPE_SYS_DESCR, sys_descr);
+  bmp_put_info_tlv(stream, BMP_INFO_TLV_TYPE_SYS_NAME, sys_name);
+  bmp_fix_common_hdr(stream);
 }
 
 static void
@@ -453,7 +437,7 @@ bmp_put_bgp_hdr(buffer *stream, const u8 msg_type, const u16 msg_length)
 }
 
 /**
- * bmp_per_peer_hdr_serialize - serializes Per-Peer Header
+ * bmp_put_per_peer_hdr - serializes Per-Peer Header
  *
  * @is_post_policy: indicate the message reflects the post-policy Adj-RIB-In
  * @peer_addr: the remote IP address associated with the TCP session
@@ -463,7 +447,7 @@ bmp_put_bgp_hdr(buffer *stream, const u8 msg_type, const u16 msg_length)
  * @ts_usec: the time in microseconds when the encapsulated routes were received
  */
 static void
-bmp_per_peer_hdr_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer)
+bmp_put_per_peer_hdr(buffer *stream, const struct bmp_peer_hdr_info *peer)
 {
   // TODO: ATM we do not support BMP_PEER_TYPE_RD_INSTANCE
   u8 peer_type = peer->global ?
@@ -478,6 +462,7 @@ bmp_per_peer_hdr_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer)
   u32 ts_sec = peer->timestamp TO_S;
   u32 ts_usec = peer->timestamp - (ts_sec S);
 
+  bmp_buffer_need(stream, BMP_PER_PEER_HDR_SIZE);
   bmp_put_u8(stream, peer_type);
   bmp_put_u8(stream, peer_flags);
   bmp_put_u64(stream, 0); // Not supported peer distinguisher
@@ -499,8 +484,8 @@ bmp_route_monitor_msg_serialize(struct bmp_proto *p, const struct bmp_peer_hdr_i
   STACK_BUFFER_INIT(stream, BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE);
 
   const size_t data_size = BMP_PER_PEER_HDR_SIZE + update_msg_length;
-  bmp_common_hdr_serialize(&stream, BMP_ROUTE_MONITOR, data_size);
-  bmp_per_peer_hdr_serialize(&stream, peer);
+  bmp_put_common_hdr(&stream, BMP_ROUTE_MONITOR, BMP_COMMON_HDR_SIZE + data_size);
+  bmp_put_per_peer_hdr(&stream, peer);
 
   size_t hdr_sz = stream.pos - stream.start;
   ASSERT_DIE(update_msg >= &p->msgbuf[hdr_sz]);
@@ -516,14 +501,10 @@ bmp_peer_up_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info *
 				const byte *sent_msg, const size_t sent_msg_length,
 				const byte *recv_msg, const size_t recv_msg_length)
 {
-  const size_t data_size =
-    BMP_PER_PEER_HDR_SIZE + BMP_PEER_UP_NOTIF_MSG_FIX_SIZE +
-    BGP_HEADER_LENGTH + sent_msg_length + BGP_HEADER_LENGTH + recv_msg_length;
-
-  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
-  bmp_common_hdr_serialize(stream, BMP_PEER_UP_NOTIF, data_size);
-  bmp_per_peer_hdr_serialize(stream, peer);
+  bmp_put_common_hdr(stream, BMP_PEER_UP_NOTIF, 0);
+  bmp_put_per_peer_hdr(stream, peer);
 
+  bmp_buffer_need(stream, BMP_PEER_UP_NOTIF_MSG_FIX_SIZE);
   bmp_put_ipa(stream, local_addr);
   bmp_put_u16(stream, local_port);
   bmp_put_u16(stream, remote_port);
@@ -531,20 +512,16 @@ bmp_peer_up_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info *
   bmp_put_data(stream, sent_msg, sent_msg_length);
   bmp_put_bgp_hdr(stream, PKT_OPEN, BGP_HEADER_LENGTH + recv_msg_length);
   bmp_put_data(stream, recv_msg, recv_msg_length);
+
+  bmp_fix_common_hdr(stream);
 }
 
 static void
 bmp_peer_down_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer,
 				  const struct bmp_peer_down_info *info)
 {
-  const size_t data_size = BMP_PER_PEER_HDR_SIZE + 1 +
-    (((info->reason == BMP_PEER_DOWN_REASON_LOCAL_BGP_NOTIFICATION) ||
-      (info->reason == BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION)) ? (BGP_HEADER_LENGTH + 2 + info->length) :
-     (info->reason == BMP_PEER_DOWN_REASON_LOCAL_NO_NOTIFICATION) ? 2 : 0);
-
-  bmp_buffer_need(stream, BMP_COMMON_HDR_SIZE + data_size);
-  bmp_common_hdr_serialize(stream, BMP_PEER_DOWN_NOTIF, data_size);
-  bmp_per_peer_hdr_serialize(stream, peer);
+  bmp_put_common_hdr(stream, BMP_PEER_DOWN_NOTIF, 0);
+  bmp_put_per_peer_hdr(stream, peer);
 
   bmp_put_u8(stream, info->reason);
 
@@ -554,7 +531,7 @@ bmp_peer_down_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info
   case BMP_PEER_DOWN_REASON_REMOTE_BGP_NOTIFICATION:;
     uint bgp_msg_length = BGP_HEADER_LENGTH + 2 + info->length;
     bmp_buffer_need(stream, bgp_msg_length);
-    bmp_put_bgp_hdr(stream, bgp_msg_length, PKT_NOTIFICATION);
+    bmp_put_bgp_hdr(stream, PKT_NOTIFICATION, bgp_msg_length);
     bmp_put_u8(stream, info->err_code);
     bmp_put_u8(stream, info->err_subcode);
     bmp_put_data(stream, info->data, info->length);
@@ -564,6 +541,8 @@ bmp_peer_down_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info
     bmp_put_u16(stream, info->fsm_code);
     break;
   }
+
+  bmp_fix_common_hdr(stream);
 }
 
 
@@ -1002,20 +981,16 @@ bmp_peer_down(const struct bgp_proto *bgp,
 }
 
 static void
-bmp_send_termination_msg(struct bmp_proto *p,
-  const enum bmp_term_reason reason)
+bmp_send_termination_msg(struct bmp_proto *p, enum bmp_term_reason reason)
 {
-  const size_t term_msg_hdr_size = BMP_TERM_INFO_TYPE_SIZE
-                                     + BMP_TERM_INFO_LEN_FIELD_SIZE
-                                     + BMP_TERM_REASON_CODE_SIZE;
-  const size_t term_msg_size = BMP_COMMON_HDR_SIZE + term_msg_hdr_size;
   buffer stream = bmp_default_buffer(p);
-  bmp_buffer_need(&stream, term_msg_size);
 
-  bmp_common_hdr_serialize(&stream, BMP_TERM_MSG, term_msg_hdr_size);
+  bmp_put_common_hdr(&stream, BMP_TERM_MSG, 0);
+  bmp_buffer_need(&stream, 6);
   bmp_put_u16(&stream, BMP_TERM_INFO_REASON);
-  bmp_put_u16(&stream, BMP_TERM_REASON_CODE_SIZE); // 2-byte code indication the reason
+  bmp_put_u16(&stream, 2); // Length of the reason
   bmp_put_u16(&stream, reason);
+  bmp_fix_common_hdr(&stream);
 
   if (p->sk->tbuf)
     bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data, p->sk->tbuf));

From a47704a53db4f088e52e43f8b24785e5777ce147 Mon Sep 17 00:00:00 2001
From: Ondrej Zajicek <santiago@crfreenet.org>
Date: Mon, 2 Dec 2024 00:00:36 +0100
Subject: [PATCH 06/15] BMP: Refactor route monitor message serialization

Instead of several levels of functions, just have two functions
(one for routes, the other for end-of-rib), this allows to create
messages in a simple linear fashion.

Also reduce three duplicite functions to construct BGP header for
BMP messages to just one.
---
 proto/bgp/packets.c |  19 +-------
 proto/bmp/bmp.c     | 103 +++++++++++++++++++++-----------------------
 2 files changed, 51 insertions(+), 71 deletions(-)

diff --git a/proto/bgp/packets.c b/proto/bgp/packets.c
index ec648231..81f0d3c4 100644
--- a/proto/bgp/packets.c
+++ b/proto/bgp/packets.c
@@ -2522,23 +2522,11 @@ bgp_create_update_bmp(struct bgp_channel *c, byte *buf, byte *end, struct bgp_bu
   return res;
 }
 
-static byte *
-bgp_bmp_prepare_bgp_hdr(byte *buf, const u16 msg_size, const u8 msg_type)
-{
-  memset(buf + BGP_MSG_HDR_MARKER_POS, 0xff, BGP_MSG_HDR_MARKER_SIZE);
-  put_u16(buf + BGP_MSG_HDR_LENGTH_POS, msg_size);
-  put_u8(buf + BGP_MSG_HDR_TYPE_POS, msg_type);
-
-  return buf + BGP_MSG_HDR_TYPE_POS + BGP_MSG_HDR_TYPE_SIZE;
-}
-
 byte *
 bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, byte *end, const net_addr *n,
 		   const struct rte *new, const struct rte_src *src)
 {
 //  struct bgp_proto *p = (void *) c->c.proto;
-  byte *pkt = buf + BGP_HEADER_LENGTH;
-
   ea_list *attrs = new ? new->attrs->eattrs : NULL;
   uint ea_size = new ? (sizeof(ea_list) + attrs->count * sizeof(eattr)) : 0;
   uint bucket_size = sizeof(struct bgp_bucket) + ea_size;
@@ -2559,12 +2547,7 @@ bgp_bmp_encode_rte(struct bgp_channel *c, byte *buf, byte *end, const net_addr *
   net_copy(px->net, n);
   add_tail(&b->prefixes, &px->buck_node);
 
-  end = bgp_create_update_bmp(c, pkt, end, b, !!new);
-
-  if (end)
-    bgp_bmp_prepare_bgp_hdr(buf, end - buf, PKT_UPDATE);
-
-  return end;
+  return bgp_create_update_bmp(c, buf, end, b, !!new);
 }
 
 #endif /* CONFIG_BMP */
diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index dc21ac09..9603c4e8 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -297,8 +297,11 @@ bmp_init_msg_serialize(buffer *stream, const char *sys_descr, const char *sys_na
 }
 
 static void
-bmp_schedule_tx_packet(struct bmp_proto *p, const byte *payload, size_t size)
+bmp_schedule_tx_packet(struct bmp_proto *p, buffer *msg)
 {
+  const byte *payload = msg->start;
+  size_t size = msg->pos - msg->start;
+
   ASSERT(p->started);
 
   while (size)
@@ -473,28 +476,6 @@ bmp_put_per_peer_hdr(buffer *stream, const struct bmp_peer_hdr_info *peer)
   bmp_put_u32(stream, ts_usec);
 }
 
-/* [4.6] Route Monitoring */
-static byte *
-bmp_route_monitor_msg_serialize(struct bmp_proto *p, const struct bmp_peer_hdr_info *peer,
-				byte *update_msg, size_t update_msg_length)
-{
-  ASSERT_DIE(update_msg < &p->msgbuf[sizeof p->msgbuf]);
-
-  buffer stream;
-  STACK_BUFFER_INIT(stream, BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE);
-
-  const size_t data_size = BMP_PER_PEER_HDR_SIZE + update_msg_length;
-  bmp_put_common_hdr(&stream, BMP_ROUTE_MONITOR, BMP_COMMON_HDR_SIZE + data_size);
-  bmp_put_per_peer_hdr(&stream, peer);
-
-  size_t hdr_sz = stream.pos - stream.start;
-  ASSERT_DIE(update_msg >= &p->msgbuf[hdr_sz]);
-
-  byte *begin = &update_msg[-hdr_sz];
-  memcpy(begin, stream.start, hdr_sz);
-  return begin;
-}
-
 static void
 bmp_peer_up_notif_msg_serialize(buffer *stream, const struct bmp_peer_hdr_info *peer,
 				const ip_addr local_addr, const u16 local_port, const u16 remote_port,
@@ -852,13 +833,17 @@ bmp_send_peer_up_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
   buffer payload = bmp_default_buffer(p);
   bmp_peer_up_notif_msg_serialize(&payload, &peer, sk->saddr, sk->sport, sk->dport,
 				  tx_data, tx_data_size, rx_data, rx_data_size);
-  bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
+  bmp_schedule_tx_packet(p, &payload);
 }
 
 static void
-bmp_route_monitor_put_update(struct bmp_proto *p, struct bmp_stream *bs, byte *data, size_t length, btime timestamp)
+bmp_route_monitor_notify(struct bmp_proto *p, struct bmp_stream *bs,
+			 const net_addr *n, const struct rte *new, const struct rte_src *src)
 {
   struct bgp_proto *bgp = bs->bgp;
+  struct bgp_channel *c = bs->sender;
+
+  btime delta_t = new ? current_time() - new->lastmod : 0;
 
   struct bmp_peer_hdr_info peer = {
     .address = bgp->remote_ip,
@@ -866,43 +851,55 @@ bmp_route_monitor_put_update(struct bmp_proto *p, struct bmp_stream *bs, byte *d
     .id = bgp->remote_id,
     .global = bmp_is_peer_global_instance(bgp),
     .policy = bmp_stream_policy(bs),
-    .timestamp = timestamp,
+    .timestamp = current_real_time() - delta_t,
   };
 
-  const byte *start = bmp_route_monitor_msg_serialize(p, &peer, data, length);
-  bmp_schedule_tx_packet(p, start, (data - start) + length);
-}
+  buffer msg = bmp_default_buffer(p);
+  bmp_put_common_hdr(&msg, BMP_ROUTE_MONITOR, 0);
+  bmp_put_per_peer_hdr(&msg, &peer);
 
-static void
-bmp_route_monitor_notify(struct bmp_proto *p, struct bmp_stream *bs,
-			 const net_addr *n, const struct rte *new, const struct rte_src *src)
-{
-  byte *bufend = &p->msgbuf[sizeof p->msgbuf];
-  byte *begin = bufend - BGP_MAX_EXT_MSG_LENGTH;
-  byte *end = bgp_bmp_encode_rte(bs->sender, begin, bufend, n, new, src);
-
-  btime delta_t = new ? current_time() - new->lastmod : 0;
-  btime timestamp = current_real_time() - delta_t;
-
-  if (end)
-    bmp_route_monitor_put_update(p, bs, begin, end - begin, timestamp);
-  else
+  bmp_buffer_need(&msg, BGP_MAX_EXT_MSG_LENGTH);
+  byte *pos = bgp_bmp_encode_rte(c, msg.pos + BGP_HEADER_LENGTH, msg.end, n, new, src);
+  if (!pos)
+  {
     log(L_WARN "%s: Cannot encode update for %N", p->p.name, n);
+    return;
+  }
+  bmp_put_bgp_hdr(&msg, PKT_UPDATE, pos - msg.pos);
+  msg.pos = pos;
+
+  bmp_fix_common_hdr(&msg);
+  bmp_schedule_tx_packet(p, &msg);
 }
 
 static void
 bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_stream *bs)
 {
-  TRACE(D_PACKETS, "Sending END-OF-RIB for %s.%s", bs->bgp->p.name, bs->sender->c.name);
+  struct bgp_proto *bgp = bs->bgp;
+  struct bgp_channel *c = bs->sender;
 
-  byte *rx_end_payload = p->msgbuf + BMP_PER_PEER_HDR_SIZE + BMP_COMMON_HDR_SIZE;
-  byte *pos = bgp_create_end_mark_(bs->sender, rx_end_payload + BGP_HEADER_LENGTH);
-  memset(rx_end_payload + BGP_MSG_HDR_MARKER_POS, 0xff,
-	 BGP_MSG_HDR_MARKER_SIZE); // BGP UPDATE MSG marker
-  put_u16(rx_end_payload + BGP_MSG_HDR_LENGTH_POS, pos - rx_end_payload);
-  put_u8(rx_end_payload + BGP_MSG_HDR_TYPE_POS, PKT_UPDATE);
+  struct bmp_peer_hdr_info peer = {
+    .address = bgp->remote_ip,
+    .as = bgp->remote_as,
+    .id = bgp->remote_id,
+    .global = bmp_is_peer_global_instance(bgp),
+    .policy = bmp_stream_policy(bs),
+    .timestamp = current_real_time(),
+  };
 
-  bmp_route_monitor_put_update(p, bs, rx_end_payload, pos - rx_end_payload, current_real_time());
+  TRACE(D_PACKETS, "Sending END-OF-RIB for %s.%s", bgp->p.name, c->c.name);
+
+  buffer msg = bmp_default_buffer(p);
+  bmp_put_common_hdr(&msg, BMP_ROUTE_MONITOR, 0);
+  bmp_put_per_peer_hdr(&msg, &peer);
+
+  bmp_buffer_need(&msg, BGP_MAX_EXT_MSG_LENGTH);
+  byte *pos = bgp_create_end_mark_(c, msg.pos + BGP_HEADER_LENGTH);
+  bmp_put_bgp_hdr(&msg, PKT_UPDATE, pos - msg.pos);
+  msg.pos = pos;
+
+  bmp_fix_common_hdr(&msg);
+  bmp_schedule_tx_packet(p, &msg);
 }
 
 static void
@@ -925,7 +922,7 @@ bmp_send_peer_down_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
 
   buffer payload = bmp_default_buffer(p);
   bmp_peer_down_notif_msg_serialize(&payload, &peer, info);
-  bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
+  bmp_schedule_tx_packet(p, &payload);
 }
 
 static void
@@ -1089,7 +1086,7 @@ bmp_startup(struct bmp_proto *p)
   /* Send initiation message */
   buffer payload = bmp_default_buffer(p);
   bmp_init_msg_serialize(&payload, p->sys_descr, p->sys_name);
-  bmp_schedule_tx_packet(p, payload.start, payload.pos - payload.start);
+  bmp_schedule_tx_packet(p, &payload);
 
   /* Send Peer Up messages */
   struct proto *peer;

From 2128c648e096e063d556cfd9805a9eed82a48702 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Wed, 4 Dec 2024 10:16:42 +0100
Subject: [PATCH 07/15] BMP: Tamed a TX buffer warning

---
 proto/bmp/bmp.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index aefb195d..40bb55f3 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -407,7 +407,7 @@ bmp_tx(struct birdsock *sk)
 {
   struct bmp_proto *p = sk->data;
 
-  struct bmp_tx_buffer *btb = SKIP_BACK(struct bmp_tx_buffer, data, sk->tbuf);
+  struct bmp_tx_buffer *btb = SKIP_BACK(struct bmp_tx_buffer, data[0], sk->tbuf);
   bmp_tx_buffer_free(p, btb);
 
   sk->tbuf = NULL;
@@ -1032,7 +1032,7 @@ bmp_send_termination_msg(struct bmp_proto *p, enum bmp_term_reason reason)
   bmp_fix_common_hdr(&stream);
 
   if (p->sk->tbuf)
-    bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data, p->sk->tbuf));
+    bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data[0], p->sk->tbuf));
 
   p->sk->tbuf = stream.start;
   if (sk_send(p->sk, stream.pos - stream.start) < 0)
@@ -1335,7 +1335,7 @@ static void
 bmp_close_socket(struct bmp_proto *p)
 {
   if (p->sk && p->sk->tbuf)
-    bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data, p->sk->tbuf));
+    bmp_tx_buffer_free(p, SKIP_BACK(struct bmp_tx_buffer, data[0], p->sk->tbuf));
 
   struct bmp_tx_buffer *btb = p->tx_pending;
   while (btb)

From d5f01efbe69020e62edf378ee9777bb1955f00fc Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Thu, 5 Dec 2024 12:12:34 +0100
Subject: [PATCH 08/15] BMP: protocol state ingestion refactoring

---
 proto/bmp/bmp.c | 89 +++++++++++++++++++++++++------------------------
 1 file changed, 45 insertions(+), 44 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 40bb55f3..44279874 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -647,18 +647,24 @@ static struct bmp_stream *
 bmp_find_stream(struct bmp_proto *p, const struct bgp_proto *bgp, u32 afi, bool policy)
 {
   ea_list *bgp_attr = proto_get_state(bgp->p.id);
-  struct bmp_stream *s = HASH_FIND(p->stream_map, HASH_STREAM, bgp_attr, bmp_stream_key(afi, policy));
 
-  while (s == NULL)
+  while (1)
   {
+    /* Is there a stream? */
+    struct bmp_stream *s = HASH_FIND(p->stream_map, HASH_STREAM, bgp_attr, bmp_stream_key(afi, policy));
+    if (s)
+      return s;
+
+    /* Maybe it emerged recently? */
     struct lfjour_item *li = lfjour_get(&p->proto_state_reader);
     if (!li)
       return NULL;
 
     bmp_process_proto_state_change(p, li);
-    s = HASH_FIND(p->stream_map, HASH_STREAM, bgp_attr, bmp_stream_key(afi, policy));
+    lfjour_release(&p->proto_state_reader, li);
+
+    /* Try again. */
   }
-  return s;
 }
 
 static struct bmp_stream *
@@ -767,17 +773,17 @@ bmp_remove_peer(struct bmp_proto *p, struct bmp_peer *bp)
   mb_free(bp);
 }
 
-static void
-bmp_peer_up_(struct bmp_proto *p, ea_list *bgp_attr, bool sync,
+static struct bmp_peer *
+bmp_peer_up_(struct bmp_proto *p, ea_list *bgp_attr,
 	     const adata *tx_open_msg, const adata *rx_open_msg,
 	     struct bgp_conn_sk_ad *sk)
 {
   if (!p->started)
-    return;
+    return NULL;
 
   struct bmp_peer *bp = bmp_find_peer(p, bgp_attr);
   if (bp)
-    return;
+    return bp;
 
   const char *name = ea_get_adata(bgp_attr, &ea_name)->data;
   TRACE(D_STATES, "Peer up for %s", name);
@@ -785,25 +791,11 @@ bmp_peer_up_(struct bmp_proto *p, ea_list *bgp_attr, bool sync,
   bp = bmp_add_peer(p, bgp_attr);
 
   bmp_send_peer_up_notif_msg(p, bgp_attr, tx_open_msg, rx_open_msg, sk);
-
-  /*
-   * We asssume peer_up() notifications are received before any route
-   * notifications from that peer. Therefore, peers established after BMP
-   * session coould be considered synced with empty RIB.
-   */
-  if (sync)
-  {
-    struct bmp_stream *bs;
-    WALK_LIST(bs, bp->streams)
-    {
-      bmp_route_monitor_end_of_rib(p, bs);
-      bs->sync = true;
-    }
-  }
+  return bp;
 }
 
-static bool
-bmp_peer_up_inout(struct bmp_proto *p, ea_list *bgp_attr, bool sync)
+static struct bmp_peer *
+bmp_peer_up_inout(struct bmp_proto *p, ea_list *bgp_attr)
 {
   int in_state = ea_get_int(bgp_attr, &ea_bgp_in_conn_state, 0);
   int out_state = ea_get_int(bgp_attr, &ea_bgp_out_conn_state, 0);
@@ -817,9 +809,7 @@ bmp_peer_up_inout(struct bmp_proto *p, ea_list *bgp_attr, bool sync)
     SKIP_BACK_DECLARE(struct bgp_conn_sk_ad, sk, ad, ea_get_adata(bgp_attr, &ea_bgp_in_conn_sk));
 
     ASSERT_DIE(loc_open && rem_open);
-    bmp_peer_up_(p, bgp_attr, sync, loc_open, rem_open, sk);
-
-    return true;
+    return bmp_peer_up_(p, bgp_attr, loc_open, rem_open, sk);
   }
 
   if (out_state == BS_ESTABLISHED)
@@ -829,12 +819,10 @@ bmp_peer_up_inout(struct bmp_proto *p, ea_list *bgp_attr, bool sync)
     SKIP_BACK_DECLARE(struct bgp_conn_sk_ad, sk, ad, ea_get_adata(bgp_attr, &ea_bgp_out_conn_sk));
 
     ASSERT_DIE(loc_open && rem_open);
-    bmp_peer_up_(p, bgp_attr, sync, loc_open, rem_open, sk);
-
-    return true;
+    return bmp_peer_up_(p, bgp_attr, loc_open, rem_open, sk);
   }
 
-  return false;
+  return NULL;
 }
 
 static bool
@@ -1191,7 +1179,7 @@ bmp_startup(struct bmp_proto *p)
     if (proto != &proto_bgp || state != PS_UP)
       continue;
 
-    bmp_peer_up_inout(p, proto_attr, false);
+    bmp_peer_up_inout(p, proto_attr);
   }
 }
 
@@ -1381,18 +1369,28 @@ bmp_process_proto_state_change(struct bmp_proto *p, struct lfjour_item *last_up)
   if (!ppu)
     return;
 
-  if (bmp_peer_up_inout(p, ppu->new, true))
-    goto done;
-
-  SKIP_BACK_DECLARE(struct bgp_session_close_ad, bscad, ad, ea_get_adata(ppu->new, &ea_bgp_close_bmp));
-  if (bscad)
+  struct bmp_peer *bp = bmp_peer_up_inout(p, ppu->new);
+  if (bp)
   {
-    bmp_peer_down_(p, ppu->new, bscad);
-    goto done;
+    /*
+     * All the peer up notifications are required to arrive before any route
+     * notifications from that peer. Therefore, peers established after BMP
+     * session are considered synced with empty RIB.
+     */
+    struct bmp_stream *bs;
+    WALK_LIST(bs, bp->streams)
+    {
+      bmp_route_monitor_end_of_rib(p, bs);
+      bs->sync = true;
+    }
+
+    return;
   }
 
-done:
-  lfjour_release(&p->proto_state_reader, last_up);
+  /* This was not a peer-up notification. It may be peer down tho. */
+  const adata *bscad = ea_get_adata(ppu->new, &ea_bgp_close_bmp);
+  if (bscad)
+    bmp_peer_down_(p, ppu->new, SKIP_BACK(struct bgp_session_close_ad, ad, bscad));
 }
 
 static void
@@ -1402,8 +1400,11 @@ bmp_proto_state_changed(void *_p)
 
   ASSERT_DIE(birdloop_inside(p->p.loop));
 
-  struct lfjour_item *last_up;
-  while (last_up = lfjour_get(&p->proto_state_reader))
+  for (
+      struct lfjour_item *last_up;
+      last_up = lfjour_get(&p->proto_state_reader);
+      lfjour_release(&p->proto_state_reader, last_up)
+      )
     bmp_process_proto_state_change(p, last_up);
 }
 

From f1a04ce190e43b2878f15a8f2e3518cb578a2981 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Thu, 5 Dec 2024 14:59:54 +0100
Subject: [PATCH 09/15] BMP: refactored lists and table locks to tlists

---
 proto/bmp/bmp.c | 62 ++++++++++++-------------------------------------
 proto/bmp/bmp.h | 34 +++++++++++++++++++--------
 2 files changed, 39 insertions(+), 57 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 44279874..0a8ac03f 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -583,12 +583,7 @@ bmp_add_table(struct bmp_proto *p, rtable *tab)
 static void
 bmp_remove_table(struct bmp_proto *p, struct bmp_table *bt)
 {
-  if (bt->channel)
-  {
-    channel_set_state(bt->channel, CS_STOP);
-    channel_set_state(bt->channel, CS_DOWN);
-  }
-    rt_export_unsubscribe(all, &bt->out_req);
+  rt_export_unsubscribe(all, &bt->out_req);
 
   HASH_REMOVE(p->table_map, HASH_TABLE, bt);
 
@@ -598,38 +593,10 @@ bmp_remove_table(struct bmp_proto *p, struct bmp_table *bt)
   mb_free(bt);
 }
 
-static inline void bmp_lock_table(struct bmp_proto *p UNUSED, struct bmp_table *bt)
-{ bt->uc++; }
-
-struct bmp_table *
+static inline struct bmp_table *
 bmp_get_table(struct bmp_proto *p, rtable *tab)
 {
-  struct bmp_table *bt = bmp_find_table(p, tab);
-  if (bt)
-  {
-   while (true) {
-      atomic_int i = bt->uc;
-      if (i == 0)
-      {
-        struct bmp_table *new = bmp_add_table(p, tab);
-        bmp_lock_table(p, new);
-        return new;
-      }
-      if (atomic_compare_exchange_strong_explicit(&bt->uc, &i, i+1, memory_order_acq_rel, memory_order_relaxed))
-        return bt;
-    }
-  }
-  struct bmp_table *new = bmp_add_table(p, tab);
-  bmp_lock_table(p, new);
-  return new;
-}
-
-static inline void bmp_unlock_table(struct bmp_proto *p, struct bmp_table *bt)
-{ atomic_int i = 1;
-  if (atomic_compare_exchange_strong_explicit(&bt->uc, &i, 0, memory_order_acq_rel, memory_order_relaxed))
-    bmp_remove_table(p, bt);
-  else
-    bt->uc--;
+  return bmp_find_table(p, tab) ?: bmp_add_table(p, tab);
 }
 
 
@@ -674,10 +641,11 @@ bmp_add_stream(struct bmp_proto *p, struct bmp_peer *bp, u32 afi, bool policy, r
   bs->bgp = bp->bgp;
   bs->key = bmp_stream_key(afi, policy);
 
-  add_tail(&bp->streams, &bs->n);
+  bmp_peer_stream_add_tail(&bp->streams, bs);
   HASH_INSERT(p->stream_map, HASH_STREAM, bs);
 
-  bs->table = bmp_get_table(p, tab);
+  struct bmp_table *bt = bmp_get_table(p, tab);
+  bmp_table_stream_add_tail(&bt->streams, bs);
 
   bs->sender = sender;
   bs->sync = false;
@@ -689,11 +657,13 @@ bmp_add_stream(struct bmp_proto *p, struct bmp_peer *bp, u32 afi, bool policy, r
 static void
 bmp_remove_stream(struct bmp_proto *p, struct bmp_stream *bs)
 {
-  rem_node(&bs->n);
+  bmp_peer_stream_rem_node(bmp_peer_stream_enlisted(bs), bs);
   HASH_REMOVE(p->stream_map, HASH_STREAM, bs);
 
-  bmp_unlock_table(p, bs->table);
-  bs->table = NULL;
+  SKIP_BACK_DECLARE(struct bmp_table, bt, streams, bmp_table_stream_enlisted(bs));
+  bmp_table_stream_rem_node(&bt->streams, bs);
+  if (EMPTY_TLIST(bmp_table_stream, &bt->streams))
+    bmp_remove_table(p, bt);
 
   mb_free(bs);
 }
@@ -723,7 +693,7 @@ bmp_add_peer(struct bmp_proto *p, ea_list *bgp_attr)
   }
   bp->bgp = bgp_attr;
 
-  init_list(&bp->streams);
+  bp->streams = (TLIST_LIST(bmp_peer_stream)) {};
 
   HASH_INSERT(p->peer_map, HASH_PEER, bp);
 
@@ -764,8 +734,7 @@ bmp_add_peer(struct bmp_proto *p, ea_list *bgp_attr)
 static void
 bmp_remove_peer(struct bmp_proto *p, struct bmp_peer *bp)
 {
-  struct bmp_stream *bs, *bs_next;
-  WALK_LIST_DELSAFE(bs, bs_next, bp->streams)
+  WALK_TLIST_DELSAFE(bmp_peer_stream, bs, &bp->streams)
     bmp_remove_stream(p, bs);
 
   HASH_REMOVE(p->peer_map, HASH_PEER, bp);
@@ -1115,7 +1084,7 @@ bmp_feed_end(struct rt_export_request *req)
 
   HASH_WALK(p->stream_map, next, bs)
   {
-    if ((bs->table == bt) && !bs->sync)
+    if ((bmp_table_stream_enlisted(bs) == &bt->streams) && !bs->sync)
     {
       bmp_route_monitor_end_of_rib(p, bs);
       bs->sync = true;
@@ -1377,8 +1346,7 @@ bmp_process_proto_state_change(struct bmp_proto *p, struct lfjour_item *last_up)
      * notifications from that peer. Therefore, peers established after BMP
      * session are considered synced with empty RIB.
      */
-    struct bmp_stream *bs;
-    WALK_LIST(bs, bp->streams)
+    WALK_TLIST(bmp_peer_stream, bs, &bp->streams)
     {
       bmp_route_monitor_end_of_rib(p, bs);
       bs->sync = true;
diff --git a/proto/bmp/bmp.h b/proto/bmp/bmp.h
index 9ab37922..283c74d0 100644
--- a/proto/bmp/bmp.h
+++ b/proto/bmp/bmp.h
@@ -12,6 +12,7 @@
 #include "nest/bird.h"
 #include "nest/protocol.h"
 #include "lib/lists.h"
+#include "lib/tlists.h"
 #include "nest/route.h"
 #include "lib/event.h"
 #include "lib/hash.h"
@@ -91,33 +92,46 @@ struct bmp_proto {
   byte msgbuf[BMP_MSGBUF_LEN];     // Buffer for preparing the messages before sending them out
 };
 
-struct bmp_peer {
-  ea_list *bgp;
-  struct bmp_peer *next;
-  list streams;
-};
-
 struct bmp_stream {
-  node n;
+  TLIST_NODE(bmp_peer_stream, struct bmp_stream) peer_node;
+  TLIST_NODE(bmp_table_stream, struct bmp_stream) table_node;
   ea_list *bgp;
   u32 key;
   bool sync;
   bool shutting_down;
   struct bmp_stream *next;
-  struct bmp_table *table;
   ea_list *sender;
   int in_pre_policy;
 };
 
+#define TLIST_PREFIX bmp_peer_stream
+#define TLIST_TYPE struct bmp_stream
+#define TLIST_ITEM peer_node
+#define TLIST_WANT_ADD_TAIL
+
+#include "lib/tlists.h"
+
+#define TLIST_PREFIX bmp_table_stream
+#define TLIST_TYPE struct bmp_stream
+#define TLIST_ITEM table_node
+#define TLIST_WANT_ADD_TAIL
+
+#include "lib/tlists.h"
+
+struct bmp_peer {
+  ea_list *bgp;
+  struct bmp_peer *next;
+  TLIST_LIST(bmp_peer_stream) streams;
+};
+
 struct bmp_table {
   rtable *table;
   struct bmp_table *next;
-  struct channel *channel;
   struct rt_export_request out_req;
   struct bmp_proto *p;
   struct rt_export_feeder in_req;
   event event;
-  atomic_int uc;
+  TLIST_LIST(bmp_table_stream) streams;
 };
 
 

From 77a4b21365ba261345b9eecd396366bd7678c5bb Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Thu, 5 Dec 2024 15:07:56 +0100
Subject: [PATCH 10/15] BMP: Dropped remnants of unused channels

---
 proto/bmp/bmp.c | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 0a8ac03f..ee0eea59 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -540,15 +540,12 @@ bmp_find_table(struct bmp_proto *p, rtable *tab)
   return HASH_FIND(p->table_map, HASH_TABLE, tab);
 }
 
-const struct channel_class channel_bmp = {
-  .channel_size =	sizeof(struct channel),
-  .config_size =	sizeof(struct channel_config),
-  /*.init =		
-  .start =		
-  .shutdown =		
-  .cleanup =		
-  .reconfigure =	*/
-};
+static void
+bmp_dump_export_req(struct rt_export_request *req)
+{
+  SKIP_BACK_DECLARE(struct bmp_table, bt, out_req, req);
+  debug("  BMP %s exporter %p\n", bt->p->p.name, req);
+}
 
 static struct bmp_table *
 bmp_add_table(struct bmp_proto *p, rtable *tab)
@@ -561,8 +558,8 @@ bmp_add_table(struct bmp_proto *p, rtable *tab)
   HASH_INSERT(p->table_map, HASH_TABLE, bt);
 
   bt->event.data = bt;
-
   bt->event.hook = bmp_check_routes;
+
   bt->out_req = (struct rt_export_request) {
     .name = mb_sprintf(p->p.pool, "%s.export", p->p.name),
     .r = (struct lfjour_recipient) {
@@ -571,7 +568,7 @@ bmp_add_table(struct bmp_proto *p, rtable *tab)
     },
     .pool = p->p.pool,
     .trace_routes = p->p.debug,
-    //.dump = channel_dump_export_req, TODO: this will crash on `dump tables` from CLI
+    .dump = bmp_dump_export_req,
     .fed = bmp_feed_end,
   };
 

From bc15f0b9c704dcac85ac471d96509e866dc5b09b Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Tue, 10 Dec 2024 10:52:52 +0100
Subject: [PATCH 11/15] Nest: fixed a race-condition between import and export

There was a leaking stack pointer to the global memory.
Fixed by making that temporary structure thread local static.
---
 nest/rt-table.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/nest/rt-table.c b/nest/rt-table.c
index e2497311..cd37d947 100644
--- a/nest/rt-table.c
+++ b/nest/rt-table.c
@@ -1964,10 +1964,8 @@ rte_recalculate(struct rtable_private *table, struct rt_import_hook *c, struct n
     }
 
   /* We need to add a spinlock sentinel to the beginning */
-  struct rte_storage local_sentinel = {
-    .flags = REF_OBSOLETE,
-    .next = old_best_stored,
-  };
+  _Thread_local static struct rte_storage local_sentinel = { .flags = REF_OBSOLETE, };
+  atomic_store_explicit(&local_sentinel.next, old_best_stored, memory_order_release);
   atomic_store_explicit(&net->routes, &local_sentinel, memory_order_release);
 
   /* Mark also the old route as obsolete. */

From d503204aa0773e847938ddd5d587b1ed20b99ed1 Mon Sep 17 00:00:00 2001
From: Katerina Kubecova <katerina.kubecova@nic.cz>
Date: Fri, 6 Dec 2024 13:16:18 +0100
Subject: [PATCH 12/15] Lockfree journal: Cleanup hook runs only when needed.

The function lfjour_cleanup_hook() was scheduled each time any of the
journal recipients reached end of a block of journal items or read all
of journal items. Because lfjour_cleanup_hook() can clean only journal
items every recipient has processed, it was often called uselessly.

This commit restricts most of the unuseful scheduling. Only some
recipients are given a token alowing them to try to schedule the
cleanup hook. When a recipient wants to schedule the cleanup hook, it
checks whether it has a token. If yes, it decrements number of tokens
the journal has given (issued_tokens) and discards its own token. If
issued_tokens reaches zero, the recipient is allowed to schedule the
cleanup hook.

There is a maximum number of tokens a journal can give to its recipients
(max_tokens). A new recipient is given a token in its init, unless the
maximum number of tokens is reached. The rest of tokens is given to
customers in lfjour_cleanup_hook().

In the cleanup hook, the issued_tokens number is increased in order to
avoid calling the hook before it finishes. Then, tokens are given to the
slowest recipients (but never to more than max_token recipients). Before
leaving lfjour_cleanup_hook(), the issued_tokens number is decreased back.
If no other tokens are given, we have to make sure the
lfjour_cleanup_hook will be called again. If every item in journal was
read by every recipient, tokens are given to random recipients. If all
recipients with tokens managed to finish until now, we give the token to
the first unfinished customer we find, or we just call the hook again.
---
 lib/lockfree.c | 159 +++++++++++++++++++++++++++++++++++++++++++++++--
 lib/lockfree.h |   3 +
 2 files changed, 158 insertions(+), 4 deletions(-)

diff --git a/lib/lockfree.c b/lib/lockfree.c
index 2d57b46a..c68899d3 100644
--- a/lib/lockfree.c
+++ b/lib/lockfree.c
@@ -38,6 +38,27 @@ void lfuc_unlock_deferred(struct deferred_call *dc)
     off / s; \
     })
 
+
+static void
+lfjour_return_cleanup_token(struct lfjour *j, struct lfjour_recipient *r)
+{
+  /* Eligible for requesting cleanup */
+  if (!(atomic_fetch_and_explicit(&r->recipient_flags, ~LFJOUR_R_LAST_RUNNER, memory_order_acq_rel) & LFJOUR_R_LAST_RUNNER))
+    return;
+
+  /* Return the lastrunner's token. */
+  u64 pings = atomic_fetch_sub_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+
+  /* Is this the last token? */
+  if (pings > 1)
+    return;
+
+  ASSERT_DIE(pings != 0);
+
+  /* No more cleanup tokens issued, request cleanup. */
+  lfjour_schedule_cleanup(j);
+}
+
 struct lfjour_item *
 lfjour_push_prepare(struct lfjour *j)
 {
@@ -228,7 +249,7 @@ void lfjour_release(struct lfjour_recipient *r, const struct lfjour_item *it)
 
   /* Releasing this export for cleanup routine */
   if (pos + 1 == end)
-  {
+{
     lfjour_debug("lfjour(%p)_release(recipient=%p) of %p, seq=%lu (end)",
 	j, r, r->cur, r->cur->seq);
   }
@@ -242,7 +263,7 @@ void lfjour_release(struct lfjour_recipient *r, const struct lfjour_item *it)
 
   /* The last block may be available to free */
   if ((pos + 1 == end) || last && (last_block != block))
-    lfjour_schedule_cleanup(j);
+    lfjour_return_cleanup_token(j, r);
 
   r->first_holding_seq = 0;
   r->cur = NULL;
@@ -299,6 +320,13 @@ lfjour_register(struct lfjour *j, struct lfjour_recipient *r)
   ASSERT_DIE(!r->cur);
 
   lfjour_recipient_add_tail(&j->recipients, r);
+
+  if (j->issued_tokens < j->max_tokens)
+  {
+    /* Cleanup hook does not run, so we are sure j->issued_tokens is not increasing */
+    atomic_fetch_add_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+    atomic_fetch_or_explicit(&r->recipient_flags, LFJOUR_R_LAST_RUNNER, memory_order_acq_rel);
+  }
 }
 
 void
@@ -311,7 +339,7 @@ lfjour_unregister(struct lfjour_recipient *r)
     lfjour_release(r, r->cur);
 
   lfjour_recipient_rem_node(&j->recipients, r);
-  lfjour_schedule_cleanup(j);
+  lfjour_return_cleanup_token(j, r);
 }
 
 static inline void lfjour_cleanup_unlock_helper(struct domain_generic **dg)
@@ -320,6 +348,110 @@ static inline void lfjour_cleanup_unlock_helper(struct domain_generic **dg)
   DG_UNLOCK(*dg);
 }
 
+static bool
+lfjour_issue_cleanup_token(struct lfjour *j, struct lfjour_recipient* rec)
+{
+  /* This function keeps the invariant that the total number of issued
+   * LFJOUR_R_LAST_RUNNER flags is the same as j->issued_tokens.
+   *
+   * Returs true if the token was successfully issued.
+   */
+
+  /* The journal is not empty and the recipient is on seq_max,
+   * so it won't call cleanup */
+  const struct lfjour_item *last = atomic_load_explicit(&rec->last, memory_order_acquire);
+  if (last && last->seq == (j->next_seq - 1))
+    return false;
+
+  /* Take a token from the pile */
+  if (atomic_fetch_add_explicit(&j->issued_tokens, 1, memory_order_acq_rel) >= j->max_tokens)
+  {
+    /* No more tokens to issue, already at max */
+    atomic_fetch_sub_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+    return false;
+  }
+
+  /* Trying to give the token */
+  if (atomic_fetch_or_explicit(&rec->recipient_flags, LFJOUR_R_LAST_RUNNER, memory_order_acq_rel) & LFJOUR_R_LAST_RUNNER)
+  {
+    /* This recipient already has the token */
+    atomic_fetch_sub_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+    return false;
+  }
+
+  /* Has't the recipient finished inbetween? (Recheck.) */
+  last = atomic_load_explicit(&rec->last, memory_order_acquire);
+  if (last && last->seq == (j->next_seq - 1))
+  {
+    /* It has! Retreat! */
+    if (atomic_fetch_and_explicit(&rec->recipient_flags, ~LFJOUR_R_LAST_RUNNER, memory_order_acq_rel) & LFJOUR_R_LAST_RUNNER)
+    {
+      /* The flag was still set, we managed to get it back safely.
+       * Now the recipient won't call the cleanup. */
+      atomic_fetch_sub_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+      return false;
+    }
+
+    /* The recipient was quick enough to finish the task and
+     * grab the flag. Now we are going to get pinged anyway
+     * so we consider the flag to be issued. */
+  }
+
+  /* Now the token is issued. */
+  return true;
+}
+
+static void
+lfjour_cleanup_done(struct lfjour *j)
+{
+  /* Returning the cleanup token. */
+  u64 pings = atomic_fetch_sub_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+
+  /* Somebody else is also holding a token, nothing to do.
+   * They'll call us when the time comes. */
+  if (pings > 1)
+    return;
+
+  ASSERT_DIE(pings == 1);
+
+  /* We'll need to know whether the journal is empty or not. */
+  struct lfjour_item *first = atomic_load_explicit(&j->first, memory_order_acquire);
+
+  /* No recipients, schedule one more cleanup if not empty */
+  if (EMPTY_TLIST(lfjour_recipient, &j->recipients))
+  {
+    if (first)
+      lfjour_schedule_cleanup(j);
+    return;
+  }
+
+  /* There are some recipients but nothing to clean.
+   * Somebody is going to wake us, let's just throw
+   * the token to the crowd. */
+  if (!first)
+  {
+    WALK_TLIST(lfjour_recipient, r, &j->recipients)
+      /* If we failed to issue a cleanup token, there is definitely
+       * somebody holding it, no more tokens needed. */
+      if (! lfjour_issue_cleanup_token(j, r))
+      {
+	ASSERT_DIE(atomic_load_explicit(&j->issued_tokens, memory_order_acquire) > 0);
+	return;
+      }
+  }
+
+  /* We have to find some recipient which has not
+   * yet finished. When we find it, it will happily
+   * accept the token so we are done. */
+  WALK_TLIST(lfjour_recipient, r, &j->recipients)
+    if (lfjour_issue_cleanup_token(j, r))
+      return;
+
+  /* Nobody needs anything, but the journal is not empty.
+   * Run cleanup again. */
+  lfjour_schedule_cleanup(j);
+}
+
 static void
 lfjour_cleanup_hook(void *_j)
 {
@@ -328,15 +460,22 @@ lfjour_cleanup_hook(void *_j)
   CLEANUP(lfjour_cleanup_unlock_helper) struct domain_generic *_locked = j->domain;
   if (_locked) DG_LOCK(_locked);
 
+  /* We consider ourselves holding one token to keep recipients
+   * from scheduling us too early. */
+  atomic_fetch_add_explicit(&j->issued_tokens, 1, memory_order_acq_rel);
+
   u64 min_seq = ~((u64) 0);
   const struct lfjour_item *last_item_to_free = NULL;
   struct lfjour_item *first = atomic_load_explicit(&j->first, memory_order_acquire);
 
   if (!first)
   {
+    lfjour_cleanup_done(j);
+
     /* Nothing to cleanup, actually, just call the done callback */
     ASSERT_DIE(EMPTY_TLIST(lfjour_block, &j->pending));
     CALL(j->cleanup_done, j, 0, ~((u64) 0));
+
     return;
   }
 
@@ -345,8 +484,14 @@ lfjour_cleanup_hook(void *_j)
     const struct lfjour_item *last = atomic_load_explicit(&r->last, memory_order_acquire);
 
     if (!last)
-      /* No last export means that the channel has exported nothing since last cleanup */
+    {
+      /* No last export means that the channel has exported nothing since
+       * last cleanup. Let's try to give them the token,
+       * and we're done anyway. */
+      lfjour_issue_cleanup_token(j, r);
+      lfjour_cleanup_done(j);
       return;
+    }
 
     else if (min_seq > last->seq)
     {
@@ -373,6 +518,9 @@ lfjour_cleanup_hook(void *_j)
 	  memory_order_acq_rel, memory_order_acquire))
     {
       lfjour_debug("lfjour(%p)_cleanup(recipient=%p): store last=NULL", j, r);
+
+      /* The most-behind-one gets the cleanup token */
+      lfjour_issue_cleanup_token(j, r);
     }
     else
     {
@@ -429,6 +577,8 @@ lfjour_cleanup_hook(void *_j)
     first = next;
   }
 
+  lfjour_cleanup_done(j);
+
   CALL(j->cleanup_done, j, orig_first_seq, first ? first->seq : ~((u64) 0));
 }
 
@@ -452,4 +602,5 @@ lfjour_init(struct lfjour *j, struct settle_config *scf)
     .hook = lfjour_cleanup_hook,
     .data = j,
   };
+  j->max_tokens = 20;
 }
diff --git a/lib/lockfree.h b/lib/lockfree.h
index ab7f7d0e..8dc8f82d 100644
--- a/lib/lockfree.h
+++ b/lib/lockfree.h
@@ -219,6 +219,7 @@ struct lfjour_recipient {
 
 enum lfjour_recipient_flags {
   LFJOUR_R_SEQ_RESET = 1,			/* Signalling of sequence number reset */
+  LFJOUR_R_LAST_RUNNER = 2,                     /* Set if this recipient is supposed to ping cleanup hook */
 };
 
 /* Defines lfjour_recipient_list */
@@ -237,6 +238,8 @@ struct lfjour {
   event announce_kick_event;			/* Kicks announce_timer */
   struct settle announce_timer;			/* Announces changes to recipients */
   event cleanup_event;				/* Runs the journal cleanup routine */
+  u64 max_tokens;				/* Maximum number of cleanup tokens to issue */
+  _Atomic u64 issued_tokens;			/* Current count of issued tokens */
 
   /* Callback on item removal from journal */
   void (*item_done)(struct lfjour *, struct lfjour_item *);

From dc692ae06a77f0e6c859411a005d0f9693d21ef7 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Mon, 9 Dec 2024 15:34:11 +0100
Subject: [PATCH 13/15] Nest: proto/channel state table naming convention
 cleanup

---
 nest/proto.c    | 68 ++++++++++++++++++++++++-------------------------
 nest/protocol.h |  8 +++---
 proto/bgp/bgp.c |  4 +--
 proto/bmp/bmp.c |  4 +--
 proto/mrt/mrt.c |  4 +--
 5 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/nest/proto.c b/nest/proto.c
index 4d9d5354..670ee506 100644
--- a/nest/proto.c
+++ b/nest/proto.c
@@ -260,21 +260,21 @@ proto_add_channel(struct proto *p, struct channel_config *cf)
     hmap_set(&ts->channel_id_map, c->id);
 
     /* The current channel state table may be too small */
-    if (c->id >= ts->length_channels)
+    if (c->id >= ts->channels_len)
     {
-      ea_list **l = mb_allocz(ts->pool, sizeof(ea_list*) * ts->length_channels * 2);
-      memcpy(l, ts->channels, sizeof(ea_list*) * ts->length_channels);
-      mb_free(ts->channels);
+      ea_list **l = mb_allocz(ts->pool, sizeof(ea_list*) * ts->channels_len * 2);
+      memcpy(l, ts->channel_states, sizeof(ea_list*) * ts->channels_len);
+      mb_free(ts->channel_states);
 
-      ts->channels = l;
-      ts->length_channels = ts->length_channels * 2;
+      ts->channel_states = l;
+      ts->channels_len = ts->channels_len * 2;
     }
 
-    ASSERT_DIE(c->id < ts->length_channels);
-    ASSERT_DIE(ts->channels[c->id] == NULL);
+    ASSERT_DIE(c->id < ts->channels_len);
+    ASSERT_DIE(ts->channel_states[c->id] == NULL);
 
     /* Set the channel info */
-    ts->channels[c->id] = ea_lookup_slow(ca, 0, EALS_IN_TABLE);
+    ts->channel_states[c->id] = ea_lookup_slow(ca, 0, EALS_IN_TABLE);
   }
 
   /* Update channel list in protocol state */
@@ -317,9 +317,9 @@ proto_remove_channel(struct proto *p UNUSED, struct channel *c)
    */
   PST_LOCKED(ts)
   {
-    ASSERT_DIE(c->id < ts->length_channels);
-    ea_free_later(ts->channels[c->id]);
-    ts->channels[c->id] = NULL;
+    ASSERT_DIE(c->id < ts->channels_len);
+    ea_free_later(ts->channel_states[c->id]);
+    ts->channel_states[c->id] = NULL;
     hmap_clear(&ts->channel_id_map, c->id);
   }
 
@@ -1348,15 +1348,15 @@ proto_new(struct proto_config *cf)
     p->id = hmap_first_zero(&tp->proto_id_map);
     hmap_set(&tp->proto_id_map, p->id);
 
-    if (p->id >= tp->length_states)
+    if (p->id >= tp->proto_len)
     {
       /* Grow the states array */
-      ea_list **new_states = mb_allocz(tp->pool, sizeof *new_states * tp->length_states * 2);
-      memcpy(new_states, tp->states, tp->length_states * sizeof *new_states);
+      ea_list **new_states = mb_allocz(tp->pool, sizeof *new_states * tp->proto_len * 2);
+      memcpy(new_states, tp->proto_states, tp->proto_len * sizeof *new_states);
 
-      mb_free(tp->states);
-      tp->states = new_states;
-      tp->length_states *= 2;
+      mb_free(tp->proto_states);
+      tp->proto_states = new_states;
+      tp->proto_len *= 2;
     }
   }
 
@@ -2140,15 +2140,15 @@ protos_build(void)
 
   PST_LOCKED(ts)
   {
-    ts->length_channels = 64;
-    ts->length_states = 32;
+    ts->channels_len = 64;
+    ts->proto_len = 32;
 
-    hmap_init(&ts->proto_id_map, p, ts->length_states); /* for proto ids. Value of proto id is the same as index of that proto in ptoto_state_table->attrs */
-    hmap_init(&ts->channel_id_map, p, ts->length_channels);
+    hmap_init(&ts->proto_id_map, p, ts->proto_len); /* for proto ids. Value of proto id is the same as index of that proto in ptoto_state_table->attrs */
+    hmap_init(&ts->channel_id_map, p, ts->channels_len);
 
     ts->pool = p;
-    ts->states = mb_allocz(p, sizeof(ea_list *) * ts->length_states);
-    ts->channels = mb_allocz(p, sizeof(ea_list *) * ts->length_channels * 2);
+    ts->proto_states = mb_allocz(p, sizeof(ea_list *) * ts->proto_len);
+    ts->channel_states = mb_allocz(p, sizeof(ea_list *) * ts->channels_len * 2);
   }
 
   /* Init proto state journal */
@@ -2939,7 +2939,7 @@ proto_journal_item_cleanup_(bool withdrawal, ea_list *old_attr)
   {
     u32 id = ea_get_int(old_attr, &ea_proto_id, 0);
     ASSERT_DIE(id);
-    ASSERT_DIE(tp->states[id] == NULL);
+    ASSERT_DIE(tp->proto_states[id] == NULL);
     hmap_clear(&tp->proto_id_map, id);
   }
 }
@@ -2955,7 +2955,7 @@ proto_journal_item_cleanup(struct lfjour * journal UNUSED, struct lfjour_item *i
 /*
  * Protocol state announcement.
  *
- * The authoritative protocol state is always stored in ts->states[p->id]
+ * The authoritative protocol state is always stored in ts->proto_states[p->id]
  * and it holds a reference. But sometimes it's too clumsy to announce all
  * protocol changes happening in a fast succession, so there is a
  * state-to-be-announced stored in the protocol itself, in p->ea_state.
@@ -3014,15 +3014,15 @@ proto_announce_state_locked(struct proto_state_table_private* ts, struct proto *
   }
 
   /* Then we check the public state */
-  ASSERT_DIE(p->id < ts->length_states);
-  ea_list *old_state = ts->states[p->id];
+  ASSERT_DIE(p->id < ts->proto_len);
+  ea_list *old_state = ts->proto_states[p->id];
 
   /* Nothing has changed? */
   if (p->ea_state == old_state)
     return;
 
   /* Set the new state */
-  ts->states[p->id] = p->ea_state ? ea_ref(p->ea_state) : NULL;
+  ts->proto_states[p->id] = p->ea_state ? ea_ref(p->ea_state) : NULL;
 
   /* Announce the new state */
   struct lfjour_item *li = lfjour_push_prepare(&proto_state_table_pub.journal);
@@ -3094,9 +3094,9 @@ channel_get_state(int id)
 {
   PST_LOCKED(ts)
   {
-    ASSERT_DIE((u32) id < ts->length_channels);
-    if (ts->channels[id])
-      return ea_ref_tmp(ts->channels[id]);
+    ASSERT_DIE((u32) id < ts->channels_len);
+    if (ts->channel_states[id])
+      return ea_ref_tmp(ts->channel_states[id]);
   }
   return NULL;
 }
@@ -3107,8 +3107,8 @@ proto_get_state(int id)
   ea_list *eal;
   PST_LOCKED(ts)
   {
-    ASSERT_DIE((u32)id < ts->length_states);
-    eal = ts->states[id];
+    ASSERT_DIE((u32)id < ts->proto_len);
+    eal = ts->proto_states[id];
   }
   if (eal)
     return ea_ref_tmp(eal);
diff --git a/nest/protocol.h b/nest/protocol.h
index 43fabdea..5526d48f 100644
--- a/nest/protocol.h
+++ b/nest/protocol.h
@@ -412,10 +412,10 @@ struct proto_state_table_private {
     PROTO_STATE_TABLE_PUBLIC;
   };
   struct proto_state_table_private **locked_at;
-  ea_list ** states;
-  ea_list ** channels;
-  u32 length_states;
-  u32 length_channels;
+  ea_list ** proto_states;
+  ea_list ** channel_states;
+  u32 proto_len;
+  u32 channels_len;
   struct hmap proto_id_map;
   struct hmap channel_id_map;
   pool *pool;
diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c
index db5eb6c5..e87ec287 100644
--- a/proto/bgp/bgp.c
+++ b/proto/bgp/bgp.c
@@ -2025,9 +2025,9 @@ bgp_channel_init(struct channel *C, struct channel_config *CF)
 
   PST_LOCKED(ts)
   {
-    ea_list *eal = ea_free_later(ts->channels[c->c.id]);
+    ea_list *eal = ea_free_later(ts->channel_states[c->c.id]);
     ea_set_attr(&eal, EA_LITERAL_EMBEDDED(&ea_bgp_afi, 0, c->afi));
-    ts->channels[c->c.id] = ea_lookup_slow(eal, 0, EALS_IN_TABLE);
+    ts->channel_states[c->c.id] = ea_lookup_slow(eal, 0, EALS_IN_TABLE);
   }
 
 }
diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index ee0eea59..e38b9764 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -702,7 +702,7 @@ bmp_add_peer(struct bmp_proto *p, ea_list *bgp_attr)
   {
     ea_list *chan_attr;
     PST_LOCKED(ts)
-      chan_attr = ts->channels[chann_ids[i]];
+      chan_attr = ts->channel_states[chann_ids[i]];
 
     if (chan_attr == NULL)
       continue;
@@ -1117,7 +1117,7 @@ bmp_startup(struct bmp_proto *p)
   /* Send Peer Up messages */
   u32 length;
   PST_LOCKED(ts) /* The size of protos field will never decrease, the inconsistency caused by growing is not important */
-    length = ts->length_states;
+    length = ts->proto_len;
 
   /* Subscribe to protocol state changes */
   p->proto_state_reader = (struct lfjour_recipient) {
diff --git a/proto/mrt/mrt.c b/proto/mrt/mrt.c
index 31e46d91..ff3b0227 100644
--- a/proto/mrt/mrt.c
+++ b/proto/mrt/mrt.c
@@ -335,9 +335,9 @@ mrt_peer_table_dump(struct mrt_table_dump_state *s)
 #ifdef CONFIG_BGP
   PST_LOCKED(ts)
   {
-    for(u32 i = 0; i < ts->length_states; i++)
+    for(u32 i = 0; i < ts->proto_len; i++)
     {
-      ea_list *eal = ts->states[i];
+      ea_list *eal = ts->proto_states[i];
       if (eal)
         ea_free_later(ea_ref(eal));
       else

From cde582977fbc568f12f8e40851def33f382259dd Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Mon, 9 Dec 2024 11:21:01 +0100
Subject: [PATCH 14/15] BMP: Never touching the BGP directly

Dropped hopefully last remnants of BMP directly accessing BGP structures.
---
 lib/route.h         |   2 +-
 lib/type.h          |   2 +-
 nest/proto.c        |   6 +-
 nest/protocol.h     |   2 -
 proto/bgp/attrs.c   |  11 ++
 proto/bgp/bgp.c     |  18 +++
 proto/bgp/bgp.h     |   4 +-
 proto/bgp/packets.c |  30 ++---
 proto/bmp/Makefile  |   2 +-
 proto/bmp/bmp.c     | 311 +++++++++++++++++++++++++-------------------
 proto/bmp/bmp.h     |  52 +++-----
 proto/bmp/map.c     | 119 -----------------
 proto/bmp/map.h     |  68 ----------
 13 files changed, 241 insertions(+), 386 deletions(-)
 delete mode 100644 proto/bmp/map.c
 delete mode 100644 proto/bmp/map.h

diff --git a/lib/route.h b/lib/route.h
index b9bbd2ae..783ef34a 100644
--- a/lib/route.h
+++ b/lib/route.h
@@ -423,7 +423,7 @@ ea_set_attr_u32(ea_list **to, const struct ea_class *def, uint flags, u64 data)
 { ea_set_attr(to, EA_LITERAL_EMBEDDED(def, flags, data)); }
 
 static inline void
-ea_set_attr_ptr(ea_list **to, const struct ea_class *def, uint flags, const void *data)
+ea_set_attr_ptr(ea_list **to, const struct ea_class *def, uint flags, void *data)
 { ea_set_attr(to, EA_LITERAL_STORE_PTR(def, flags, data)); }
 
 static inline void
diff --git a/lib/type.h b/lib/type.h
index e42c6728..9244ea40 100644
--- a/lib/type.h
+++ b/lib/type.h
@@ -24,7 +24,7 @@ union bval {
   };											\
   const struct adata *ptr;	/* Generic attribute data inherited from eattrs */	\
   const struct adata *ad;     	/* Generic attribute data inherited from filters */	\
-  const void * v_ptr;       /* Stored pointer */ \
+  void * v_ptr;			/* Stored pointer */ \
 
   BVAL_ITEMS;
 };
diff --git a/nest/proto.c b/nest/proto.c
index 670ee506..5a46514b 100644
--- a/nest/proto.c
+++ b/nest/proto.c
@@ -2146,6 +2146,10 @@ protos_build(void)
     hmap_init(&ts->proto_id_map, p, ts->proto_len); /* for proto ids. Value of proto id is the same as index of that proto in ptoto_state_table->attrs */
     hmap_init(&ts->channel_id_map, p, ts->channels_len);
 
+    /* Zeros should be reserved for easier undef manipulation */
+    hmap_set(&ts->proto_id_map, 0);
+    hmap_set(&ts->channel_id_map, 0);
+
     ts->pool = p;
     ts->proto_states = mb_allocz(p, sizeof(ea_list *) * ts->proto_len);
     ts->channel_states = mb_allocz(p, sizeof(ea_list *) * ts->channels_len * 2);
@@ -2992,7 +2996,7 @@ struct proto_announce_state_deferred {
   struct proto *p;
 };
 
-void
+static void
 proto_announce_state_locked(struct proto_state_table_private* ts, struct proto *p, ea_list *new_state)
 {
   ASSERT_DIE(birdloop_inside(p->loop));
diff --git a/nest/protocol.h b/nest/protocol.h
index 5526d48f..171a0a5b 100644
--- a/nest/protocol.h
+++ b/nest/protocol.h
@@ -442,9 +442,7 @@ struct proto_pending_update {
   ea_list *new, *old;
 };
 
-void proto_announce_state_locked(struct proto_state_table_private *ts, struct proto *p, ea_list *attr);
 void proto_announce_state(struct proto *p, ea_list *attr);
-
 void proto_announce_state_later_internal(struct proto *p, ea_list *attr);
 #if 0
 #define proto_announce_state_later(p, a) ( log(L_INFO "proto_announce_state_later(%s (%p), %p) at %s:%d", (p)->name, (p), (a), __FILE__, __LINE__), proto_announce_state_later_internal((p), (a)) )
diff --git a/proto/bgp/attrs.c b/proto/bgp/attrs.c
index 0e843b50..957fe108 100644
--- a/proto/bgp/attrs.c
+++ b/proto/bgp/attrs.c
@@ -1318,6 +1318,16 @@ struct ea_class ea_bgp_as4_out_conn = {
   .type = T_INT,
 };
 
+struct ea_class ea_bgp_extended_next_hop = {
+  .name = "bgp_extended_next_hop",
+  .type = T_INT,
+};
+
+struct ea_class ea_bgp_add_path_rx = {
+  .name = "bgp_add_path_rx",
+  .type = T_INT,
+};
+
 void
 bgp_register_attrs(void)
 {
@@ -1340,6 +1350,7 @@ bgp_register_attrs(void)
 
   EA_REGISTER_ALL(
       &ea_bgp_rem_id, &ea_bgp_rem_as, &ea_bgp_loc_as, &ea_bgp_rem_ip, &ea_bgp_peer_type, &ea_bgp_afi,
+      &ea_bgp_extended_next_hop, &ea_bgp_add_path_rx,
       &ea_bgp_in_conn_local_open_msg, &ea_bgp_out_conn_local_open_msg, &ea_bgp_in_conn_remote_open_msg,
       &ea_bgp_out_conn_remote_open_msg, &ea_bgp_close_bmp, &ea_bgp_as4_session,
       &ea_bgp_state_startup, &ea_bgp_in_conn_state, &ea_bgp_out_conn_state,
diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c
index e87ec287..ea9392ff 100644
--- a/proto/bgp/bgp.c
+++ b/proto/bgp/bgp.c
@@ -801,6 +801,24 @@ bgp_conn_enter_established_state(struct bgp_conn *conn)
       c->c.ra_mode = RA_ACCEPTED;
     else
       c->c.ra_mode = RA_OPTIMAL;
+
+    ea_list *state = NULL;
+    if (c->ext_next_hop)
+      ea_set_attr_u32(&state, &ea_bgp_extended_next_hop, 0, 1);
+    if (c->add_path_rx)
+      ea_set_attr_u32(&state, &ea_bgp_add_path_rx, 0, 1);
+
+    if (state)
+    {
+      ea_list *sb = state;
+      while (sb->next)
+	sb = sb->next;
+
+      PST_LOCKED(ts) {
+	sb->next = ea_free_later(ts->channel_states[c->c.id]);
+	ts->channel_states[c->c.id] = ea_lookup_slow(state, 0, EALS_IN_TABLE);
+      }
+    }
   }
 
   p->afi_map = mb_alloc(p->p.pool, num * sizeof(u32));
diff --git a/proto/bgp/bgp.h b/proto/bgp/bgp.h
index 05aae612..c12942ec 100644
--- a/proto/bgp/bgp.h
+++ b/proto/bgp/bgp.h
@@ -508,7 +508,6 @@ struct bgp_export_state {
 };
 
 struct bgp_write_state {
-  struct bgp_proto *proto;
   struct bgp_ptx_private *ptx;
   struct linpool *pool;
 
@@ -703,7 +702,7 @@ static inline struct bgp_proto *bgp_rte_proto(const rte *rte)
     SKIP_BACK(struct bgp_proto, p.sources, rte->src->owner) : NULL;
 }
 
-byte * bgp_bmp_encode_rte(ea_list *c, struct bgp_proto *bgp_p, byte *buf, byte *end, const struct rte *new);
+byte * bgp_bmp_encode_rte(ea_list *channel_state, byte *buf, byte *end, const struct rte *new);
 
 #define BGP_AIGP_METRIC		1
 #define BGP_AIGP_MAX		U64(0xffffffffffffffff)
@@ -722,6 +721,7 @@ bgp_total_aigp_metric(const rte *e)
 extern struct ea_class ea_bgp_rem_id, ea_bgp_rem_as, ea_bgp_loc_as, ea_bgp_rem_ip, ea_bgp_peer_type, ea_bgp_afi,
       ea_bgp_in_conn_local_open_msg, ea_bgp_out_conn_local_open_msg, ea_bgp_in_conn_remote_open_msg,
       ea_bgp_out_conn_remote_open_msg, ea_bgp_close_bmp, ea_bgp_close_bmp_set, ea_bgp_as4_session,
+      ea_bgp_extended_next_hop, ea_bgp_add_path_rx,
       ea_bgp_state_startup, ea_bgp_in_conn_state, ea_bgp_out_conn_state,
       ea_bgp_in_conn_sk, ea_bgp_out_conn_sk, ea_bgp_as4_out_conn, ea_bgp_as4_in_conn;
 
diff --git a/proto/bgp/packets.c b/proto/bgp/packets.c
index 157843a6..f236eeab 100644
--- a/proto/bgp/packets.c
+++ b/proto/bgp/packets.c
@@ -2480,32 +2480,27 @@ bgp_create_mp_unreach(struct bgp_write_state *s, struct bgp_bucket *buck, byte *
 #ifdef CONFIG_BMP
 
 static byte *
-bgp_create_update_bmp(ea_list *channel_ea, struct bgp_proto *bgp_p, byte *buf, byte *end, struct bgp_bucket *buck, bool update)
+bgp_create_update_bmp(ea_list *channel_ea, byte *buf, byte *end, struct bgp_bucket *buck, bool update)
 {
-  struct bgp_channel *c;
-  u32 c_id = ea_get_int(channel_ea, &ea_channel_id, 0);
-  BGP_WALK_CHANNELS(bgp_p, c)
-    if (c->c.id == c_id)
-      break;
-
   byte *res = NULL;
 
-  struct bgp_caps *peer = bgp_p->conn->remote_caps;
-  const struct bgp_af_caps *rem = bgp_find_af_caps(peer, c->afi);
-
   struct bgp_ptx_private ptx = {
     .bmp = 1,
-    .c = c,
   };
 
+  u32 afi = ea_get_int(channel_ea, &ea_bgp_afi, 0);
+  ASSERT_DIE(afi);
+
+  const struct bgp_af_desc *desc = bgp_get_af_desc(afi);
+  ASSERT_DIE(desc);
+
   struct bgp_write_state s = {
-    .proto = bgp_p,
     .ptx = &ptx,
     .pool = tmp_linpool,
-    .mp_reach = (c->afi != BGP_AF_IPV4) || rem->ext_next_hop,
+    .mp_reach = (afi != BGP_AF_IPV4) || ea_get_int(channel_ea, &ea_bgp_extended_next_hop, 0),
     .as4_session = 1,
-    .add_path = c->add_path_rx,
-    .mpls = c->desc->mpls,
+    .add_path = ea_get_int(channel_ea, &ea_bgp_add_path_rx, 0),
+    .mpls = desc->mpls,
     .ignore_non_bgp_attrs = 1,
   };
 
@@ -2526,7 +2521,7 @@ bgp_create_update_bmp(ea_list *channel_ea, struct bgp_proto *bgp_p, byte *buf, b
 }
 
 byte *
-bgp_bmp_encode_rte(ea_list *c, struct bgp_proto *bgp_p, byte *buf, byte *end, const struct rte *new)
+bgp_bmp_encode_rte(ea_list *c, byte *buf, byte *end, const struct rte *new)
 {
   uint ea_size = new->attrs ? (sizeof(ea_list) + new->attrs->count * sizeof(eattr)) : 0;
   uint prefix_size = sizeof(struct bgp_prefix) + new->net->length;
@@ -2548,7 +2543,7 @@ bgp_bmp_encode_rte(ea_list *c, struct bgp_proto *bgp_p, byte *buf, byte *end, co
   px->ni = NET_TO_INDEX(new->net);
   add_tail(&b->prefixes, &px->buck_node);
 
-  end = bgp_create_update_bmp(c, bgp_p, buf, end, b, !!new->attrs);
+  end = bgp_create_update_bmp(c, buf, end, b, !!new->attrs);
 
   lp_restore(tmp_linpool, tmpp);
 
@@ -2577,7 +2572,6 @@ again:
 
   /* Initialize write state */
   struct bgp_write_state s = {
-    .proto = p,
     .ptx = ptx,
     .pool = tmp_linpool,
     .mp_reach = (c->afi != BGP_AF_IPV4) || c->ext_next_hop,
diff --git a/proto/bmp/Makefile b/proto/bmp/Makefile
index e34f3822..1fb0ce43 100644
--- a/proto/bmp/Makefile
+++ b/proto/bmp/Makefile
@@ -1,4 +1,4 @@
-src := bmp.c map.c
+src := bmp.c
 obj := $(src-o-files)
 $(all-daemon)
 $(cf-local)
diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index e38b9764..d5e546d2 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -29,7 +29,6 @@
  */
 
 #include "proto/bmp/bmp.h"
-#include "proto/bmp/map.h"
 
 #include <sys/socket.h>
 #include <sys/time.h>
@@ -57,17 +56,15 @@
 #include "nest/iface.h"
 #include "nest/route.h"
 
-#define HASH_PEER_KEY(n)		n->bgp
+#define HASH_PEER_KEY(n)		n->info.proto_id
 #define HASH_PEER_NEXT(n)		n->next
 #define HASH_PEER_EQ(b1,b2)		b1 == b2
-#define HASH_PEER_FN(b)			ptr_hash(b)
+#define HASH_PEER_FN(b)			u32_hash(b)
 
-#define BMP_STREAM_KEY_POLICY		0x100
-
-#define HASH_STREAM_KEY(n)		n->bgp, n->key
+#define HASH_STREAM_KEY(n)		&n->info
 #define HASH_STREAM_NEXT(n)		n->next
-#define HASH_STREAM_EQ(b1,k1,b2,k2)	b1 == b2 && k1 == k2
-#define HASH_STREAM_FN(b,k)		ptr_hash(b) ^ u32_hash(k)
+#define HASH_STREAM_EQ(k1,k2)		((k1)->channel_id == (k2)->channel_id) && ((k1)->afi == (k2)->afi) && ((k1)->mode == (k2)->mode)
+#define HASH_STREAM_FN(k)		u32_hash((k)->channel_id) ^ u32_hash((k)->afi) ^ u32_hash((k)->mode)
 
 #define HASH_TABLE_KEY(n)		n->table
 #define HASH_TABLE_NEXT(n)		n->next
@@ -107,7 +104,7 @@ struct bmp_peer_hdr_info {
   u32 as;
   u32 id;
   bool global;
-  bool policy;
+  enum bmp_stream_policy policy;
   bool no_as4;
   btime timestamp;
 };
@@ -220,8 +217,6 @@ static void
 bmp_send_peer_up_notif_msg(struct bmp_proto *p, ea_list *bgp,
     const adata *tx_data, const adata *rx_data, struct bgp_conn_sk_ad *sk);
 
-static void bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_stream *bs);
-
 // Stores TX data
 struct bmp_tx_buffer {
   struct bmp_tx_buffer *next;
@@ -463,7 +458,7 @@ bmp_put_per_peer_hdr(buffer *stream, const struct bmp_peer_hdr_info *peer)
 
   u8 peer_flags =
     (ipa_is_ip6(peer->address) ? BMP_PEER_FLAG_V_IPV6 : 0) |
-    (peer->policy ? BMP_PEER_FLAG_L_POLICY : 0) |
+    ((peer->policy == BMP_STREAM_POST_POLICY) ? BMP_PEER_FLAG_L_POLICY : 0) |
     (peer->no_as4 ? BMP_PEER_FLAG_A_NO_AS4 : 0);
 
   u32 ts_sec = peer->timestamp TO_S;
@@ -601,21 +596,13 @@ bmp_get_table(struct bmp_proto *p, rtable *tab)
  *	BMP streams
  */
 
-static inline u32 bmp_stream_key(u32 afi, bool policy)
-{ return afi ^ (policy ? BMP_STREAM_KEY_POLICY : 0); }
-
-static inline bool bmp_stream_policy(struct bmp_stream *bs)
-{ return !!(bs->key & BMP_STREAM_KEY_POLICY); }
-
 static struct bmp_stream *
-bmp_find_stream(struct bmp_proto *p, const struct bgp_proto *bgp, u32 afi, bool policy)
+bmp_find_stream(struct bmp_proto *p, struct bmp_stream_info *bsi)
 {
-  ea_list *bgp_attr = proto_get_state(bgp->p.id);
-
-  while (1)
+  while (true)
   {
     /* Is there a stream? */
-    struct bmp_stream *s = HASH_FIND(p->stream_map, HASH_STREAM, bgp_attr, bmp_stream_key(afi, policy));
+    struct bmp_stream *s = HASH_FIND(p->stream_map, HASH_STREAM, bsi);
     if (s)
       return s;
 
@@ -632,22 +619,15 @@ bmp_find_stream(struct bmp_proto *p, const struct bgp_proto *bgp, u32 afi, bool
 }
 
 static struct bmp_stream *
-bmp_add_stream(struct bmp_proto *p, struct bmp_peer *bp, u32 afi, bool policy, rtable *tab, ea_list *sender, int in_pre_policy)
+bmp_add_stream(struct bmp_proto *p, struct bmp_peer *bp, struct bmp_table *bt, const struct bmp_stream_info *bsi)
 {
   struct bmp_stream *bs = mb_allocz(p->p.pool, sizeof(struct bmp_stream));
-  bs->bgp = bp->bgp;
-  bs->key = bmp_stream_key(afi, policy);
+  bs->info = *bsi;
 
+  bmp_table_stream_add_tail(&bt->streams, bs);
   bmp_peer_stream_add_tail(&bp->streams, bs);
   HASH_INSERT(p->stream_map, HASH_STREAM, bs);
 
-  struct bmp_table *bt = bmp_get_table(p, tab);
-  bmp_table_stream_add_tail(&bt->streams, bs);
-
-  bs->sender = sender;
-  bs->sync = false;
-  bs->in_pre_policy = in_pre_policy;
-
   return bs;
 }
 
@@ -671,58 +651,63 @@ bmp_remove_stream(struct bmp_proto *p, struct bmp_stream *bs)
  */
 
 static struct bmp_peer *
-bmp_find_peer(struct bmp_proto *p, ea_list *bgp_attr)
+bmp_find_peer(struct bmp_proto *p, const struct bmp_peer_info *bpi)
 {
-  return HASH_FIND(p->peer_map, HASH_PEER, bgp_attr);
+  return HASH_FIND(p->peer_map, HASH_PEER, bpi->proto_id);
 }
 
 static struct bmp_peer *
-bmp_add_peer(struct bmp_proto *p, ea_list *bgp_attr)
+bmp_add_peer(struct bmp_proto *p, struct bmp_peer_info *bpi, ea_list **cached_channels)
 {
-  struct bmp_peer *bp;
-  if (DG_IS_LOCKED(p->p.pool->domain))
-    bp = mb_allocz(p->p.pool, sizeof(struct bmp_peer));
-  else
-  {
-    DG_LOCK(p->p.pool->domain);
-    bp = mb_allocz(p->p.pool, sizeof(struct bmp_peer));
-    DG_UNLOCK(p->p.pool->domain);
-  }
-  bp->bgp = bgp_attr;
-
-  bp->streams = (TLIST_LIST(bmp_peer_stream)) {};
-
+  struct bmp_peer *bp = mb_allocz(p->p.pool, sizeof(struct bmp_peer));
+  bp->info = *bpi;
+  ea_ref(bpi->proto_state);
   HASH_INSERT(p->peer_map, HASH_PEER, bp);
 
-  const adata *channels_adata = ea_get_adata(bgp_attr, &ea_proto_channel_list);
-  int id_count = channels_adata->length / sizeof(u32);
-  u32 *chann_ids = (u32 *) channels_adata->data;
+  const adata *channels_adata = ea_get_adata(bpi->proto_state, &ea_proto_channel_list);
 
-  for (int i = 0; i < id_count; i++)
+  if (!cached_channels)
   {
-    ea_list *chan_attr;
-    PST_LOCKED(ts)
-      chan_attr = ts->channel_states[chann_ids[i]];
+    u32 max_channel_id;
+    ASSERT_DIE(int_set_max(channels_adata, &max_channel_id));
+    cached_channels = tmp_alloc((max_channel_id + 1) * sizeof(ea_list *));
 
-    if (chan_attr == NULL)
+    PST_LOCKED(ts)
+      for (u32 pos = 0, val; int_set_walk(channels_adata, &pos, &val); )
+      {
+	ASSERT_DIE(val < ts->channels_len);
+	cached_channels[val] = ts->channel_states[val] ? ea_ref_tmp(ts->channel_states[val]) : NULL;
+      }
+  }
+
+  struct bmp_stream_info bsi;
+  for (u32 pos = 0; int_set_walk(channels_adata, &pos, &bsi.channel_id); )
+  {
+    if (!(bsi.channel_state = cached_channels[bsi.channel_id]))
       continue;
 
-    rtable *ch_table = (rtable *) ea_get_ptr(chan_attr, &ea_rtable, 0);
-    int in_keep = ea_get_int(chan_attr, &ea_in_keep, 0);
+    bsi.afi = ea_get_int(bsi.channel_state, &ea_bgp_afi, 0);
+    bsi.channel_name = ea_get_adata(bsi.channel_state, &ea_name)->data;
 
-    if (p->monitoring_rib.in_pre_policy && ch_table)
+    struct bmp_table *bt = bmp_get_table(p, ea_get_ptr(bsi.channel_state, &ea_rtable, NULL));
+ 
+    if (p->monitoring_rib.in_pre_policy)
     {
-      if (in_keep == RIK_PREFILTER)
-        bmp_add_stream(p, bp, ea_get_int(chan_attr, &ea_bgp_afi, 0), false, ch_table, chan_attr, 1);
+      bsi.mode = BMP_STREAM_PRE_POLICY;
+      if ((ea_get_int(bsi.channel_state, &ea_in_keep, 0) & RIK_PREFILTER) == RIK_PREFILTER)
+        bmp_add_stream(p, bp, bt, &bsi);
       else
         log(L_WARN "%s: Failed to request pre-policy for %s.%s, import table disabled",
 	    p->p.name,
-	    ea_get_adata(bgp_attr, &ea_name)->data,
-	    ea_get_adata(chan_attr, &ea_name)->data);
+	    bpi->proto_name,
+	    bsi.channel_name);
     }
 
-    if (p->monitoring_rib.in_post_policy && ch_table)
-      bmp_add_stream(p, bp, ea_get_int(chan_attr, &ea_bgp_afi, 0), true, ch_table, chan_attr, 0);
+    if (p->monitoring_rib.in_post_policy)
+    {
+      bsi.mode = BMP_STREAM_POST_POLICY;
+      bmp_add_stream(p, bp, bt, &bsi);
+    }
   }
 
   return bp;
@@ -735,57 +720,62 @@ bmp_remove_peer(struct bmp_proto *p, struct bmp_peer *bp)
     bmp_remove_stream(p, bs);
 
   HASH_REMOVE(p->peer_map, HASH_PEER, bp);
+  ea_free_later(bp->info.proto_state);
 
   mb_free(bp);
 }
 
 static struct bmp_peer *
-bmp_peer_up_(struct bmp_proto *p, ea_list *bgp_attr,
+bmp_peer_up_(struct bmp_proto *p, struct bmp_peer_info *bpi, ea_list **cached_channels,
 	     const adata *tx_open_msg, const adata *rx_open_msg,
 	     struct bgp_conn_sk_ad *sk)
 {
   if (!p->started)
     return NULL;
 
-  struct bmp_peer *bp = bmp_find_peer(p, bgp_attr);
+  struct bmp_peer *bp = bmp_find_peer(p, bpi);
   if (bp)
+  {
+    /* Update the locally cached proto_state */
+    ea_free_later(bp->info.proto_state);
+    bp->info.proto_state = ea_ref(bpi->proto_state);
     return bp;
+  }
 
-  const char *name = ea_get_adata(bgp_attr, &ea_name)->data;
-  TRACE(D_STATES, "Peer up for %s", name);
+  TRACE(D_STATES, "Peer up for %s", bpi->proto_name);
 
-  bp = bmp_add_peer(p, bgp_attr);
+  bp = bmp_add_peer(p, bpi, cached_channels);
 
-  bmp_send_peer_up_notif_msg(p, bgp_attr, tx_open_msg, rx_open_msg, sk);
+  bmp_send_peer_up_notif_msg(p, bpi->proto_state, tx_open_msg, rx_open_msg, sk);
   return bp;
 }
 
 static struct bmp_peer *
-bmp_peer_up_inout(struct bmp_proto *p, ea_list *bgp_attr)
+bmp_peer_up_inout(struct bmp_proto *p, struct bmp_peer_info *bpi, ea_list **cached_channels)
 {
-  int in_state = ea_get_int(bgp_attr, &ea_bgp_in_conn_state, 0);
-  int out_state = ea_get_int(bgp_attr, &ea_bgp_out_conn_state, 0);
+  int in_state = ea_get_int(bpi->proto_state, &ea_bgp_in_conn_state, 0);
+  int out_state = ea_get_int(bpi->proto_state, &ea_bgp_out_conn_state, 0);
 
   if (in_state == BS_ESTABLISHED)
   {
     ASSERT_DIE(out_state != BS_ESTABLISHED);
 
-    const adata *loc_open = ea_get_adata(bgp_attr, &ea_bgp_in_conn_local_open_msg);
-    const adata *rem_open = ea_get_adata(bgp_attr, &ea_bgp_in_conn_remote_open_msg);
-    SKIP_BACK_DECLARE(struct bgp_conn_sk_ad, sk, ad, ea_get_adata(bgp_attr, &ea_bgp_in_conn_sk));
+    const adata *loc_open = ea_get_adata(bpi->proto_state, &ea_bgp_in_conn_local_open_msg);
+    const adata *rem_open = ea_get_adata(bpi->proto_state, &ea_bgp_in_conn_remote_open_msg);
+    SKIP_BACK_DECLARE(struct bgp_conn_sk_ad, sk, ad, ea_get_adata(bpi->proto_state, &ea_bgp_in_conn_sk));
 
     ASSERT_DIE(loc_open && rem_open);
-    return bmp_peer_up_(p, bgp_attr, loc_open, rem_open, sk);
+    return bmp_peer_up_(p, bpi, cached_channels, loc_open, rem_open, sk);
   }
 
   if (out_state == BS_ESTABLISHED)
   {
-    const adata *loc_open = ea_get_adata(bgp_attr, &ea_bgp_out_conn_local_open_msg);
-    const adata *rem_open = ea_get_adata(bgp_attr, &ea_bgp_out_conn_remote_open_msg);
-    SKIP_BACK_DECLARE(struct bgp_conn_sk_ad, sk, ad, ea_get_adata(bgp_attr, &ea_bgp_out_conn_sk));
+    const adata *loc_open = ea_get_adata(bpi->proto_state, &ea_bgp_out_conn_local_open_msg);
+    const adata *rem_open = ea_get_adata(bpi->proto_state, &ea_bgp_out_conn_remote_open_msg);
+    SKIP_BACK_DECLARE(struct bgp_conn_sk_ad, sk, ad, ea_get_adata(bpi->proto_state, &ea_bgp_out_conn_sk));
 
     ASSERT_DIE(loc_open && rem_open);
-    return bmp_peer_up_(p, bgp_attr, loc_open, rem_open, sk);
+    return bmp_peer_up_(p, bpi, cached_channels, loc_open, rem_open, sk);
   }
 
   return NULL;
@@ -825,19 +815,14 @@ bmp_send_peer_up_notif_msg(struct bmp_proto *p, ea_list *bgp,
 }
 
 static void
-bmp_route_monitor_notify(struct bmp_proto *p, struct bgp_proto *bgp_p, u32 afi, bool policy, const rte *new, ea_list *old)
+bmp_route_monitor_notify(struct bmp_proto *p, struct bmp_peer *bp, struct bmp_stream *bs, const rte *new, ea_list *old)
 {
   /* Idempotent update */
   if ((old == new->attrs) || old && new->attrs && ea_same(old, new->attrs))
     return;
 
-  /* No stream, probably flushed already */
-  struct bmp_stream *bs = bmp_find_stream(p, bgp_p, afi, policy);
-  if (!bs)
-    return;
-
-  ea_list *bgp = bs->bgp;
-  ea_list *c = bs->sender;
+  ea_list *bgp = bp->info.proto_state;
+  ea_list *c = bs->info.channel_state;
 
   btime delta_t = new ? current_time() - new->lastmod : 0;
 
@@ -846,7 +831,7 @@ bmp_route_monitor_notify(struct bmp_proto *p, struct bgp_proto *bgp_p, u32 afi,
     .as = ea_get_int(bgp, &ea_bgp_rem_as, 0),
     .id = ea_get_int(bgp, &ea_bgp_rem_id, 0),
     .global = bmp_is_peer_global_instance(bgp),
-    .policy = bmp_stream_policy(bs),
+    .policy = bs->info.mode,
     .timestamp = current_real_time() - delta_t,
   };
 
@@ -855,7 +840,7 @@ bmp_route_monitor_notify(struct bmp_proto *p, struct bgp_proto *bgp_p, u32 afi,
   bmp_put_per_peer_hdr(&msg, &peer);
 
   bmp_buffer_need(&msg, BGP_MAX_EXT_MSG_LENGTH);
-  byte *pos = bgp_bmp_encode_rte(c, bgp_p, msg.pos + BGP_HEADER_LENGTH, msg.end, new);
+  byte *pos = bgp_bmp_encode_rte(c, msg.pos + BGP_HEADER_LENGTH, msg.end, new);
   if (!pos)
   {
     log(L_WARN "%s: Cannot encode update for %N", p->p.name, new->net);
@@ -869,17 +854,17 @@ bmp_route_monitor_notify(struct bmp_proto *p, struct bgp_proto *bgp_p, u32 afi,
 }
 
 static void
-bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_stream *bs)
+bmp_route_monitor_end_of_rib(struct bmp_proto *p, struct bmp_peer *bp, struct bmp_stream *bs)
 {
-  ea_list *bgp = bs->bgp;
-  ea_list *c = bs->sender;
+  ea_list *bgp = bp->info.proto_state;
+  ea_list *c = bs->info.channel_state;
 
   struct bmp_peer_hdr_info peer = {
     .address = ea_get_ip(bgp, &ea_bgp_rem_ip, IPA_NONE),
     .as = ea_get_int(bgp, &ea_bgp_rem_as, 0),
     .id = ea_get_int(bgp, &ea_bgp_rem_id, 0),
     .global = bmp_is_peer_global_instance(bgp),
-    .policy = bmp_stream_policy(bs),
+    .policy = bs->info.mode,
     .timestamp = current_real_time(),
   };
 
@@ -920,7 +905,7 @@ bmp_send_peer_down_notif_msg(struct bmp_proto *p, ea_list *bgp,
     .as = ea_get_int(bgp, &ea_bgp_rem_as, 0),
     .id = ea_get_int(bgp, &ea_bgp_rem_id, 0),
     .global = bmp_is_peer_global_instance(bgp),
-    .policy = false,		// Hardcoded pre-policy Adj-RIB-In
+    .policy = BMP_STREAM_PRE_POLICY,		// Hardcoded pre-policy Adj-RIB-In
     .no_as4 = !as4_session,
     .timestamp = 0,		// No timestamp provided
   };
@@ -931,16 +916,16 @@ bmp_send_peer_down_notif_msg(struct bmp_proto *p, ea_list *bgp,
 }
 
 static void
-bmp_peer_down_(struct bmp_proto *p, ea_list *bgp, struct bgp_session_close_ad *bscad)
+bmp_peer_down_(struct bmp_proto *p, struct bmp_peer_info *bpi, struct bgp_session_close_ad *bscad)
 {
   if (!p->started)
     return;
 
-  struct bmp_peer *bp = bmp_find_peer(p, bgp);
+  struct bmp_peer *bp = bmp_find_peer(p, bpi);
   if (!bp)
     return;
 
-  TRACE(D_STATES, "Peer down for %s", ea_find(bgp, &ea_name)->u.ad->data);
+  TRACE(D_STATES, "Peer down for %s", bpi->proto_name);
 
   struct bmp_peer_down_info info = {
     .err_code = bscad->notify_code,
@@ -967,7 +952,7 @@ bmp_peer_down_(struct bmp_proto *p, ea_list *bgp, struct bgp_session_close_ad *b
     break;
   }
 
-  bmp_send_peer_down_notif_msg(p, bgp, &info);
+  bmp_send_peer_down_notif_msg(p, bpi->proto_state, &info);
 
   bmp_remove_peer(p, bp);
 }
@@ -999,26 +984,39 @@ bmp_split_policy(struct bmp_proto *p, const rte *new, const rte *old)
 {
   rte loc = *(new ?: old);
 
-  struct proto *rte_proto = (struct proto*) SKIP_BACK(struct proto, sources, loc.src->owner);
-  struct bgp_proto *bgp = (struct bgp_proto *) rte_proto;
-  struct bgp_channel *src_ch = SKIP_BACK(struct bgp_channel, c.in_req, loc.sender->req);
-
   /* Ignore piped routes */
-  if (src_ch->c.proto != rte_proto)
+  if (loc.generation)
     return;
 
   /* Ignore non-BGP routes */
-  if (rte_proto->proto != &proto_bgp)
+  if (rt_get_source_attr(&loc) != RTS_BGP)
     return;
 
+  /* Get the protocol and channel information. Here we _can_ access
+   * the protocol and channel structures but only to read the ID. */
+  SKIP_BACK_DECLARE(struct channel, c, in_req, loc.sender->req);
+  struct bmp_peer_info bpi = {
+    .proto_id = c->proto->id,
+  };
+  struct bmp_peer *bp = bmp_find_peer(p, &bpi);
+
+  struct bmp_stream_info bsi = {
+    .channel_id = c->id,
+  };
+
   /* Checking the pre policy */
   if (p->monitoring_rib.in_pre_policy)
   {
-    /* Compute the pre policy attributes */
+    /* Compute the pre policy route attributes */
     loc.attrs = new ? ea_strip_to(new->attrs, BIT32_ALL(EALS_PREIMPORT)) : NULL;
     ea_list *old_attrs = old ? ea_strip_to(old->attrs, BIT32_ALL(EALS_PREIMPORT)) : NULL;
 
-    bmp_route_monitor_notify(p, bgp, src_ch->afi, false, &loc, old_attrs);
+    bsi.mode = BMP_STREAM_PRE_POLICY;
+    struct bmp_stream *bs = bmp_find_stream(p, &bsi);
+    if (!bs)
+      return;
+
+    bmp_route_monitor_notify(p, bp, bs, &loc, old_attrs);
   }
 
   /* Checking the post policy */
@@ -1028,7 +1026,12 @@ bmp_split_policy(struct bmp_proto *p, const rte *new, const rte *old)
     loc.attrs = new ? ea_normalize(new->attrs, 0) : NULL;
     ea_list *old_attrs = old ? ea_normalize(old->attrs, 0) : NULL;
 
-    bmp_route_monitor_notify(p, bgp, src_ch->afi, true, &loc, old_attrs);
+    bsi.mode = BMP_STREAM_POST_POLICY;
+    struct bmp_stream *bs = bmp_find_stream(p, &bsi);
+    if (!bs)
+      return;
+
+    bmp_route_monitor_notify(p, bp, bs, &loc, old_attrs);
   }
 }
 
@@ -1075,19 +1078,17 @@ bmp_feed_end(struct rt_export_request *req)
    * Unsynced streams are added in one moment during BMP session establishment,
    * therefore we can assume that all unsynced streams (for given channel)
    * already received full feed now and are synced.
-   *
-   * TODO: Use more efficent way to find bmp_stream from bmp_table
    */
 
-  HASH_WALK(p->stream_map, next, bs)
+  WALK_TLIST(bmp_table_stream, bs, &bt->streams)
   {
-    if ((bmp_table_stream_enlisted(bs) == &bt->streams) && !bs->sync)
+    if (!bs->sync)
     {
-      bmp_route_monitor_end_of_rib(p, bs);
+      SKIP_BACK_DECLARE(struct bmp_peer, bp, streams, bmp_peer_stream_enlisted(bs));
+      bmp_route_monitor_end_of_rib(p, bp, bs);
       bs->sync = true;
     }
   }
-  HASH_WALK_END;
 }
 
 
@@ -1114,11 +1115,6 @@ bmp_startup(struct bmp_proto *p)
   bmp_init_msg_serialize(&payload, p->sys_descr, p->sys_name);
   bmp_schedule_tx_packet(p, &payload);
 
-  /* Send Peer Up messages */
-  u32 length;
-  PST_LOCKED(ts) /* The size of protos field will never decrease, the inconsistency caused by growing is not important */
-    length = ts->proto_len;
-
   /* Subscribe to protocol state changes */
   p->proto_state_reader = (struct lfjour_recipient) {
     .event = &p->proto_state_changed,
@@ -1133,19 +1129,55 @@ bmp_startup(struct bmp_proto *p)
   proto_states_subscribe(&p->proto_state_reader);
 
   /* Load protocol states */
-  for (u32 i = 0; i < length; i++)
+  u32 max_proto_id, max_channel_id;
+  PST_LOCKED(tp)
   {
-    ea_list *proto_attr = proto_get_state(i);
-    if (proto_attr == NULL)
-      continue;
+    max_proto_id = tp->proto_len;
+    max_channel_id = tp->channels_len;
+  }
 
-    struct protocol *proto = (struct protocol *) ea_get_ptr(proto_attr, &ea_protocol_type, 0);
-    const int state = ea_get_int(proto_attr, &ea_state, 0);
+  /* We can unlock here as the states array never shrinks
+   * and growing will be resolved by reading the journal. */
 
-    if (proto != &proto_bgp || state != PS_UP)
-      continue;
+  u32 *protos_picked = tmp_allocz(BIRD_ALIGN(max_proto_id, 32) * sizeof *protos_picked);
 
-    bmp_peer_up_inout(p, proto_attr);
+  /* Copy relevant protocol states */
+  ea_list **pst = tmp_alloc(max_proto_id * sizeof *pst);
+  uint *proto_ids = tmp_alloc(max_proto_id * sizeof *proto_ids);
+  u32 pst_count = 0;
+
+  ea_list **cst = tmp_alloc(max_channel_id * sizeof *cst);
+
+  PST_LOCKED(ts)
+  {
+    for (u32 i = 0; i < max_proto_id; i++)
+      if (ea_get_ptr(ts->proto_states[i], &ea_protocol_type, NULL) == &proto_bgp)
+      {
+	BIT32_SET(protos_picked, i);
+	pst[pst_count] = ea_ref_tmp(ts->proto_states[i]);
+	proto_ids[pst_count] = i;
+	pst_count++;
+      }
+    for (u32 i = 0; i < max_channel_id; i++)
+    {
+      u32 proto_id = ea_get_int(ts->channel_states[i], &ea_proto_id, 0);
+      if (BIT32_TEST(protos_picked, proto_id))
+	cst[i] = ea_ref_tmp(ts->channel_states[i]);
+      else
+	cst[i] = NULL;
+    }
+  }
+
+  /* Send Peer Up messages */
+  for (u32 i = 0; i < pst_count; i++)
+  {
+    struct bmp_peer_info bpi = {
+      .proto_state = pst[i],
+      .proto_id = proto_ids[i],
+      .proto_name = ea_get_adata(pst[i], &ea_name)->data,
+    };
+
+    bmp_peer_up_inout(p, &bpi, cst);
   }
 }
 
@@ -1331,11 +1363,18 @@ bmp_postconfig(struct proto_config *CF)
 static void
 bmp_process_proto_state_change(struct bmp_proto *p, struct lfjour_item *last_up)
 {
-  struct proto_pending_update *ppu = SKIP_BACK(struct proto_pending_update, li, last_up);
-  if (!ppu)
+  if (!last_up)
     return;
 
-  struct bmp_peer *bp = bmp_peer_up_inout(p, ppu->new);
+  SKIP_BACK_DECLARE(struct proto_pending_update, ppu, li, last_up);
+
+  struct bmp_peer_info bpi = {
+    .proto_state = ppu->new,
+    .proto_id = ea_get_int(ppu->new, &ea_proto_id, 0),
+    .proto_name = ea_get_adata(ppu->new, &ea_name)->data,
+  };
+
+  struct bmp_peer *bp = bmp_peer_up_inout(p, &bpi, NULL);
   if (bp)
   {
     /*
@@ -1345,7 +1384,7 @@ bmp_process_proto_state_change(struct bmp_proto *p, struct lfjour_item *last_up)
      */
     WALK_TLIST(bmp_peer_stream, bs, &bp->streams)
     {
-      bmp_route_monitor_end_of_rib(p, bs);
+      bmp_route_monitor_end_of_rib(p, bp, bs);
       bs->sync = true;
     }
 
@@ -1355,7 +1394,7 @@ bmp_process_proto_state_change(struct bmp_proto *p, struct lfjour_item *last_up)
   /* This was not a peer-up notification. It may be peer down tho. */
   const adata *bscad = ea_get_adata(ppu->new, &ea_bgp_close_bmp);
   if (bscad)
-    bmp_peer_down_(p, ppu->new, SKIP_BACK(struct bgp_session_close_ad, ad, bscad));
+    bmp_peer_down_(p, &bpi, SKIP_BACK(struct bgp_session_close_ad, ad, bscad));
 }
 
 static void
diff --git a/proto/bmp/bmp.h b/proto/bmp/bmp.h
index 283c74d0..63ece572 100644
--- a/proto/bmp/bmp.h
+++ b/proto/bmp/bmp.h
@@ -18,7 +18,6 @@
 #include "lib/hash.h"
 #include "lib/socket.h"
 #include "proto/bgp/bgp.h"
-#include "proto/bmp/map.h"
 
 // Max length of MIB-II description object
 #define MIB_II_STR_LEN 255
@@ -53,10 +52,6 @@ struct bmp_config {
   uint tx_pending_limit;	      // Maximum on pending TX buffer count
 };
 
-/* Forward declarations */
-struct bgp_proto;
-struct bmp_proto;
-
 struct bmp_proto {
   struct proto p;                  // Parent proto
   const struct bmp_config *cf;     // Shortcut to BMP configuration
@@ -75,7 +70,6 @@ struct bmp_proto {
   u16 station_port;                // Monitoring station TCP port
   struct monitoring_rib monitoring_rib;
   // Below fields are for internal use
-  // struct bmp_peer_map bgp_peers;   // Stores 'bgp_proto' structure per BGP peer
   struct bmp_tx_buffer *tx_pending;// This buffer waits for socket to flush
   struct bmp_tx_buffer *tx_last;   // This buffer is the last to flush 
   uint tx_pending_count;	   // How many buffers waiting for flush
@@ -95,13 +89,19 @@ struct bmp_proto {
 struct bmp_stream {
   TLIST_NODE(bmp_peer_stream, struct bmp_stream) peer_node;
   TLIST_NODE(bmp_table_stream, struct bmp_stream) table_node;
-  ea_list *bgp;
-  u32 key;
   bool sync;
   bool shutting_down;
   struct bmp_stream *next;
-  ea_list *sender;
-  int in_pre_policy;
+  struct bmp_stream_info {
+    u32 channel_id;
+    ea_list *channel_state;
+    const char *channel_name;
+    u32 afi;
+    enum bmp_stream_policy {
+      BMP_STREAM_PRE_POLICY = 1,
+      BMP_STREAM_POST_POLICY,
+    } mode;
+  } info;
 };
 
 #define TLIST_PREFIX bmp_peer_stream
@@ -119,8 +119,12 @@ struct bmp_stream {
 #include "lib/tlists.h"
 
 struct bmp_peer {
-  ea_list *bgp;
   struct bmp_peer *next;
+  struct bmp_peer_info {
+    u32 proto_id;
+    ea_list *proto_state;
+    const char *proto_name;
+  } info;
   TLIST_LIST(bmp_peer_stream) streams;
 };
 
@@ -134,30 +138,4 @@ struct bmp_table {
   TLIST_LIST(bmp_table_stream) streams;
 };
 
-
-#ifdef CONFIG_BMP
-
-/**
- * bmp_peer_up - send notification that BGP peer connection is established
- */
-void
-bmp_peer_up(struct ea_list *bgp,
-	    const byte *tx_open_msg, uint tx_open_length,
-	    const byte *rx_open_msg, uint rx_open_length);
-
-/**
- * bmp_peer_down - send notification that BGP peer connection is not in
- * established state
- */
-void
-bmp_peer_down(const struct bgp_proto *bgp, int err_class, int code, int subcode, const byte *data, int length);
-
-
-#else /* BMP build disabled */
-
-static inline void bmp_peer_up(struct bgp_proto *bgp UNUSED, const byte *tx_open_msg UNUSED, uint tx_open_length UNUSED, const byte *rx_open_msg UNUSED, uint rx_open_length UNUSED) { }
-static inline void bmp_peer_down(const struct bgp_proto *bgp UNUSED, const int err_class UNUSED, int code UNUSED, int subcode UNUSED, const byte *data UNUSED, int length UNUSED) { }
-
-#endif /* CONFIG_BMP */
-
 #endif /* _BIRD_BMP_H_ */
diff --git a/proto/bmp/map.c b/proto/bmp/map.c
deleted file mode 100644
index 16e714fd..00000000
--- a/proto/bmp/map.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- *	BIRD -- The BGP Monitoring Protocol (BMP)
- *
- *	(c) 2020 Akamai Technologies, Inc. (Pawel Maslanka, pmaslank@akamai.com)
- *
- *	Can be freely distributed and used under the terms of the GNU GPL.
- */
-
-#include "proto/bmp/map.h"
-
-/* Peer Index Table */
-#define PEER_KEY(n) (n)->peer_as, (n)->peer_ip
-#define PEER_NEXT(n) (n)->next
-#define PEER_EQ(as1,ip1,as2,ip2) \
-  (as1) == (as2) && ipa_equal(ip1, ip2)
-#define PEER_FN(as,ip) ipa_hash(ip)
-
-#define PEER_REHASH bmp_peer_rehash
-#define PEER_PARAMS /8, *2, 2, 2, 6, 20
-
-HASH_DEFINE_REHASH_FN(PEER, struct bmp_peer_map_key)
-
-#define PEER_INIT_ORDER 6
-
-void
-bmp_peer_map_init(struct bmp_peer_map *map, pool *mpool)
-{
-  map->mpool = mpool;
-  HASH_INIT(map->peer_hash, map->mpool, PEER_INIT_ORDER);
-}
-
-struct bmp_peer_map_key
-bmp_peer_map_key_create(const ip_addr peer_ip, const u32 peer_as)
-{
-  struct bmp_peer_map_key key;
-  key.next = NULL;
-  key.peer_ip = peer_ip;
-  key.peer_as = peer_as;
-
-  return key;
-}
-
-void
-bmp_peer_map_flush(struct bmp_peer_map *map)
-{
-  struct bmp_peer_map_entry *entry;
-  HASH_WALK_DELSAFE(map->peer_hash, next, e)
-  {
-    entry = (struct bmp_peer_map_entry *) e;
-    mb_free(entry->data.buf);
-    HASH_DELETE(map->peer_hash, PEER, PEER_KEY(&entry->key));
-    mb_free(entry);
-  }
-  HASH_WALK_DELSAFE_END;
-
-  HASH_MAY_RESIZE_DOWN(map->peer_hash, PEER, map->mpool);
-}
-
-void
-bmp_peer_map_free(struct bmp_peer_map *map)
-{
-  bmp_peer_map_flush(map);
-  HASH_FREE(map->peer_hash);
-}
-
-void
-bmp_peer_map_insert(struct bmp_peer_map *map, const struct bmp_peer_map_key key,
-  const byte *data, const size_t data_size)
-{
-  struct bmp_peer_map_entry *entry
-    = (void *) HASH_FIND(map->peer_hash, PEER, PEER_KEY(&key));
-
-  if (entry)
-  {
-    mb_free(entry->data.buf);
-    entry->data.buf = mb_alloc(map->mpool, data_size);
-    memcpy(entry->data.buf, data, data_size);
-    entry->data.buf_size = data_size;
-    return;
-  }
-
-  entry = mb_alloc(map->mpool, sizeof (struct bmp_peer_map_entry));
-  entry->data.buf = mb_alloc(map->mpool, data_size);
-  memcpy(entry->data.buf, data, data_size);
-  entry->data.buf_size = data_size;
-  entry->key = key;
-  HASH_INSERT2(map->peer_hash, PEER, map->mpool, &entry->key);
-}
-
-void
-bmp_peer_map_remove(struct bmp_peer_map *map, const struct bmp_peer_map_key key)
-{
-  struct bmp_peer_map_entry *entry
-    = (void *) HASH_DELETE(map->peer_hash, PEER, PEER_KEY(&key));
-
-  if (!entry)
-    return;
-
-  mb_free(entry->data.buf);
-  mb_free(entry);
-}
-
-const struct bmp_peer_map_entry *
-bmp_peer_map_get(struct bmp_peer_map *map, const struct bmp_peer_map_key key)
-{
-  return (struct bmp_peer_map_entry *) HASH_FIND(map->peer_hash, PEER, PEER_KEY(&key));
-}
-
-void
-bmp_peer_map_walk(const struct bmp_peer_map *map, bmp_peer_map_walk_action action, void *arg)
-{
-  struct bmp_peer_map_entry *entry;
-  HASH_WALK_FILTER(map->peer_hash, next, e, _)
-  {
-    entry = (struct bmp_peer_map_entry *) e;
-    action(entry->key, entry->data.buf, entry->data.buf_size, arg);
-  }
-  HASH_WALK_FILTER_END;
-}
diff --git a/proto/bmp/map.h b/proto/bmp/map.h
deleted file mode 100644
index 8e7ea695..00000000
--- a/proto/bmp/map.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- *	BIRD -- The BGP Monitoring Protocol (BMP)
- *
- *	(c) 2020 Akamai Technologies, Inc. (Pawel Maslanka, pmaslank@akamai.com)
- *
- *	Can be freely distributed and used under the terms of the GNU GPL.
- */
-
-/**
- * This map implementation binds peer IP address as container key with custom data.
- */
-#ifndef _BIRD_BMP_MAP_H_
-#define _BIRD_BMP_MAP_H_
-
-#include "nest/bird.h"
-#include "lib/hash.h"
-#include "lib/resource.h"
-
-struct bmp_peer_map_key {
-  struct bmp_peer_map_key *next;
-  ip_addr peer_ip;
-  u32 peer_as;
-};
-
-struct bmp_peer_map_data {
-  void *buf;
-  size_t buf_size;
-};
-
-struct bmp_peer_map_entry {
-  struct bmp_peer_map_key key;
-  struct bmp_peer_map_data data;
-};
-
-struct bmp_peer_map {
-  pool *mpool;                             // Memory pool for peer entries in peer_hash
-  HASH(struct bmp_peer_map_key) peer_hash; // Hash for peers to find the index
-};
-
-void
-bmp_peer_map_init(struct bmp_peer_map *map, pool *mpool);
-
-struct bmp_peer_map_key
-bmp_peer_map_key_create(const ip_addr peer_ip, const u32 peer_as);
-
-void
-bmp_peer_map_free(struct bmp_peer_map *map);
-
-void
-bmp_peer_map_flush(struct bmp_peer_map *map);
-
-void
-bmp_peer_map_insert(struct bmp_peer_map *map, const struct bmp_peer_map_key key,
-  const byte *data, const size_t data_size);
-
-void
-bmp_peer_map_remove(struct bmp_peer_map *map, const struct bmp_peer_map_key key);
-
-const struct bmp_peer_map_entry *
-bmp_peer_map_get(struct bmp_peer_map *map, const struct bmp_peer_map_key key);
-
-typedef void (*bmp_peer_map_walk_action)(const struct bmp_peer_map_key key,
-					 const byte *data, const size_t data_size, void *arg);
-
-void
-bmp_peer_map_walk(const struct bmp_peer_map *map, bmp_peer_map_walk_action action, void *arg);
-
-#endif /* _BIRD_BMP_MAP_H_ */

From cc0c8ce99203ee0a79501d013db96ce8c799fad2 Mon Sep 17 00:00:00 2001
From: Maria Matejka <mq@ucw.cz>
Date: Tue, 10 Dec 2024 22:31:53 +0100
Subject: [PATCH 15/15] BMP: Fixing corner cases

Protocol state propagation collision, shutdown collision.
---
 proto/bmp/bmp.c | 44 +++++++++++++++++++++++++++++++++++---------
 proto/bmp/bmp.h |  5 ++---
 2 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index d5e546d2..39a3d4fe 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -576,6 +576,7 @@ static void
 bmp_remove_table(struct bmp_proto *p, struct bmp_table *bt)
 {
   rt_export_unsubscribe(all, &bt->out_req);
+  ev_postpone(&bt->event);
 
   HASH_REMOVE(p->table_map, HASH_TABLE, bt);
 
@@ -597,7 +598,7 @@ bmp_get_table(struct bmp_proto *p, rtable *tab)
  */
 
 static struct bmp_stream *
-bmp_find_stream(struct bmp_proto *p, struct bmp_stream_info *bsi)
+bmp_get_stream(struct bmp_proto *p, struct bmp_stream_info *bsi)
 {
   while (true)
   {
@@ -639,7 +640,9 @@ bmp_remove_stream(struct bmp_proto *p, struct bmp_stream *bs)
 
   SKIP_BACK_DECLARE(struct bmp_table, bt, streams, bmp_table_stream_enlisted(bs));
   bmp_table_stream_rem_node(&bt->streams, bs);
-  if (EMPTY_TLIST(bmp_table_stream, &bt->streams))
+  if (EMPTY_TLIST(bmp_table_stream, &bt->streams) && !bt->out_req.cur)
+    /* If out_req.cur, then we are called from bmp_check_routes()
+     * and therefore the table will be removed in the tail position there. */
     bmp_remove_table(p, bt);
 
   mb_free(bs);
@@ -656,6 +659,28 @@ bmp_find_peer(struct bmp_proto *p, const struct bmp_peer_info *bpi)
   return HASH_FIND(p->peer_map, HASH_PEER, bpi->proto_id);
 }
 
+static struct bmp_peer *
+bmp_get_peer(struct bmp_proto *p, const struct bmp_peer_info *bpi)
+{
+  while (true)
+  {
+    /* Is there a peer? */
+    struct bmp_peer *bp = bmp_find_peer(p, bpi);
+    if (bp)
+      return bp;
+
+    /* Maybe it emerged recently? */
+    struct lfjour_item *li = lfjour_get(&p->proto_state_reader);
+    if (!li)
+      return NULL;
+
+    bmp_process_proto_state_change(p, li);
+    lfjour_release(&p->proto_state_reader, li);
+
+    /* Try again. */
+  }
+}
+
 static struct bmp_peer *
 bmp_add_peer(struct bmp_proto *p, struct bmp_peer_info *bpi, ea_list **cached_channels)
 {
@@ -998,7 +1023,7 @@ bmp_split_policy(struct bmp_proto *p, const rte *new, const rte *old)
   struct bmp_peer_info bpi = {
     .proto_id = c->proto->id,
   };
-  struct bmp_peer *bp = bmp_find_peer(p, &bpi);
+  struct bmp_peer *bp = bmp_get_peer(p, &bpi);
 
   struct bmp_stream_info bsi = {
     .channel_id = c->id,
@@ -1012,7 +1037,7 @@ bmp_split_policy(struct bmp_proto *p, const rte *new, const rte *old)
     ea_list *old_attrs = old ? ea_strip_to(old->attrs, BIT32_ALL(EALS_PREIMPORT)) : NULL;
 
     bsi.mode = BMP_STREAM_PRE_POLICY;
-    struct bmp_stream *bs = bmp_find_stream(p, &bsi);
+    struct bmp_stream *bs = bmp_get_stream(p, &bsi);
     if (!bs)
       return;
 
@@ -1027,7 +1052,7 @@ bmp_split_policy(struct bmp_proto *p, const rte *new, const rte *old)
     ea_list *old_attrs = old ? ea_normalize(old->attrs, 0) : NULL;
 
     bsi.mode = BMP_STREAM_POST_POLICY;
-    struct bmp_stream *bs = bmp_find_stream(p, &bsi);
+    struct bmp_stream *bs = bmp_get_stream(p, &bsi);
     if (!bs)
       return;
 
@@ -1065,6 +1090,10 @@ bmp_check_routes(void *bt_)
 	break;
     }
   }
+
+  /* Remove if deleted */
+  if (EMPTY_TLIST(bmp_table_stream, &bt->streams))
+    bmp_remove_table(p, bt);
 }
 
 static void
@@ -1363,9 +1392,6 @@ bmp_postconfig(struct proto_config *CF)
 static void
 bmp_process_proto_state_change(struct bmp_proto *p, struct lfjour_item *last_up)
 {
-  if (!last_up)
-    return;
-
   SKIP_BACK_DECLARE(struct proto_pending_update, ppu, li, last_up);
 
   struct bmp_peer_info bpi = {
@@ -1477,9 +1503,9 @@ bmp_shutdown(struct proto *P)
   {
     bmp_send_termination_msg(p, BMP_TERM_REASON_ADM);
     bmp_down(p);
-    bmp_close_socket(p);
   }
 
+  bmp_close_socket(p);
   p->sock_err = 0;
 
   return PS_FLUSH;
diff --git a/proto/bmp/bmp.h b/proto/bmp/bmp.h
index 63ece572..0fd70009 100644
--- a/proto/bmp/bmp.h
+++ b/proto/bmp/bmp.h
@@ -129,11 +129,10 @@ struct bmp_peer {
 };
 
 struct bmp_table {
-  rtable *table;
   struct bmp_table *next;
-  struct rt_export_request out_req;
   struct bmp_proto *p;
-  struct rt_export_feeder in_req;
+  rtable *table;
+  struct rt_export_request out_req;
   event event;
   TLIST_LIST(bmp_table_stream) streams;
 };