From d4ea5619176e8ca9b1e486ec43b038d386d97cb4 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Mon, 3 Jun 2024 14:23:41 +0200 Subject: [PATCH] Common parts of the ROA updater moved into the table Channel is now just subscribing to yet another journal announcing digested tries from the ROA table. Creating tries in every channel on-the-fly was too slow to handle and it ate obnoxious amounts of memory. Instead, the tries are constructed directly in the table and the channels are notified with the completed tries. The delayed export-release mechanism is used to keep the tries allocated until routes get reloaded. --- lib/bitmap.h | 1 + nest/config.Y | 2 +- nest/proto.c | 156 +++++++++++----------------------------------- nest/protocol.h | 4 -- nest/route.h | 21 ++++++- nest/rt-table.c | 140 ++++++++++++++++++++++++++++++++++++++++- proto/bgp/attrs.c | 2 +- 7 files changed, 194 insertions(+), 132 deletions(-) diff --git a/lib/bitmap.h b/lib/bitmap.h index 01bb65b6..303618d1 100644 --- a/lib/bitmap.h +++ b/lib/bitmap.h @@ -41,6 +41,7 @@ static inline void bmap_clear(struct bmap *b, uint n) BIT32_CLR(b->data, n); } +#define BMAP_WALK(_b, _n) for (uint _max = bmap_max((_b)), _n = 0; _n < _max; _n++) if (!BIT32_TEST((_b)->data, _n)) ; else struct hmap { diff --git a/nest/config.Y b/nest/config.Y index 878383cf..b3255ba6 100644 --- a/nest/config.Y +++ b/nest/config.Y @@ -283,6 +283,7 @@ table_opt: this_table->cork_threshold.high = $4; } | EXPORT SETTLE TIME settle { this_table->export_settle = $4; } | ROUTE REFRESH EXPORT SETTLE TIME settle { this_table->export_rr_settle = $6; } + | ROA SETTLE TIME settle { this_table->roa_settle = $4; } ; table_opts: @@ -370,7 +371,6 @@ channel_item_: | RECEIVE LIMIT limit_spec { this_channel->rx_limit = $3; } | IMPORT LIMIT limit_spec { this_channel->in_limit = $3; } | EXPORT LIMIT limit_spec { this_channel->out_limit = $3; } - | ROA SETTLE TIME settle { this_channel->roa_settle = $4; } | PREFERENCE expr { this_channel->preference = $2; check_u16($2); } | IMPORT KEEP FILTERED bool { if ($4) diff --git a/nest/proto.c b/nest/proto.c index fb3f6a81..3493b239 100644 --- a/nest/proto.c +++ b/nest/proto.c @@ -389,89 +389,44 @@ proto_remove_channels(struct proto *p) struct roa_subscription { node roa_node; - struct settle settle; struct channel *c; rtable *tab; - struct rt_export_request reader; - event update_event; - struct f_trie *trie; void (*refeed_hook)(struct channel *, struct rt_feeding_request *); + struct lfjour_recipient digest_recipient; + event update_event; + struct rt_feeding_request rfr; }; static void channel_roa_reload_done(struct rt_feeding_request *req) { - rfree(req->prefilter.trie->lp); + SKIP_BACK_DECLARE(struct roa_subscription, s, rfr, req); + lfjour_release(&s->digest_recipient); + ev_send(proto_work_list(s->c->proto), &s->update_event); /* FIXME: this should reset import/export filters if ACTION BLOCK */ } static void -channel_roa_changed(struct settle *se) +channel_roa_changed(void *_s) { - SKIP_BACK_DECLARE(struct roa_subscription, s, settle, se); - struct channel *c = s->c; + struct roa_subscription *s = _s; - CD(c, "Feeding triggered by RPKI change"); + if (s->digest_recipient.cur) + return; - /* Setup feeding request */ - struct rt_feeding_request *rfr = lp_alloc(s->trie->lp, sizeof *rfr); - *rfr = (struct rt_feeding_request) { + if (!lfjour_get(&s->digest_recipient)) + return; + + SKIP_BACK_DECLARE(struct roa_digest, rd, li, s->digest_recipient.cur); + s->rfr = (struct rt_feeding_request) { .prefilter = { .mode = TE_ADDR_TRIE, - .trie = s->trie, + .trie = rd->trie, }, .done = channel_roa_reload_done, }; - /* Prepare new trie */ - s->trie = f_new_trie(lp_new(c->proto->pool), 0); - - /* Actually request the feed */ - s->refeed_hook(c, rfr); -} - -static void -channel_roa_update_net(struct roa_subscription *s, const net_addr *net) -{ - switch (net->type) - { - case NET_ROA4: - trie_add_prefix(s->trie, net, net_pxlen(net), 32); - break; - case NET_ROA6: - trie_add_prefix(s->trie, net, net_pxlen(net), 128); - break; - default: - bug("ROA table sent us a non-roa export"); - } - - settle_kick(&s->settle, s->c->proto->loop); -} - -static void -channel_roa_update(void *_s) -{ - struct roa_subscription *s = _s; - - RT_EXPORT_WALK(&s->reader, u) - switch (u->kind) - { - case RT_EXPORT_STOP: - bug("Main table export stopped"); - break; - - case RT_EXPORT_FEED: - if (u->feed->count_routes) - channel_roa_update_net(s, u->feed->block[0].net); - break; - - case RT_EXPORT_UPDATE: - /* Only switched ROA from one source to another */ - if (!u->update->new || !u->update->old) - channel_roa_update_net(s, u->update->new ? u->update->new->net : u->update->old->net); - break; - - } + s->refeed_hook(s->c, &s->rfr); } static inline void (*channel_roa_reload_hook(int dir))(struct channel *, struct rt_feeding_request *) @@ -479,18 +434,6 @@ static inline void (*channel_roa_reload_hook(int dir))(struct channel *, struct return dir ? channel_reimport : channel_refeed; } -static void -channel_dump_roa_req(struct rt_export_request *req) -{ - SKIP_BACK_DECLARE(struct roa_subscription, s, reader, req); - struct channel *c = s->c; - - debug(" Channel %s.%s ROA %s change notifier request %p\n", - c->proto->name, c->name, - (s->refeed_hook == channel_roa_reload_hook(1)) ? "import" : "export", - req); -} - static int channel_roa_is_subscribed(struct channel *c, rtable *tab, int dir) { @@ -513,30 +456,23 @@ channel_roa_subscribe(struct channel *c, rtable *tab, int dir) struct roa_subscription *s = mb_allocz(c->proto->pool, sizeof(struct roa_subscription)); *s = (struct roa_subscription) { - .settle = SETTLE_INIT(&c->roa_settle, channel_roa_changed, NULL), - .refeed_hook = channel_roa_reload_hook(dir), .c = c, - .trie = f_new_trie(lp_new(c->proto->pool), 0), .tab = tab, - .update_event = { - .hook = channel_roa_update, - .data = s, + .refeed_hook = channel_roa_reload_hook(dir), + .digest_recipient = { + .target = proto_work_list(c->proto), + .event = &s->update_event, }, - .reader = { - .name = mb_sprintf(c->proto->pool, "%s.%s.roa-%s.%s", - c->proto->name, c->name, dir ? "in" : "out", tab->name), - .r = { - .target = proto_work_list(c->proto), - .event = &s->update_event, - }, - .pool = c->proto->pool, - .trace_routes = c->debug | c->proto->debug, - .dump = channel_dump_roa_req, + .update_event = { + .hook = channel_roa_changed, + .data = s, }, }; add_tail(&c->roa_subscriptions, &s->roa_node); - rt_export_subscribe(tab, best, &s->reader); + RT_LOCK(tab, t); + rt_lock_table(t); + lfjour_register(&t->roa_digest->digest, &s->digest_recipient); } static void @@ -544,18 +480,17 @@ channel_roa_unsubscribe(struct roa_subscription *s) { struct channel *c = s->c; - rt_export_unsubscribe(best, &s->reader); - settle_cancel(&s->settle); - s->settle.hook = NULL; + RT_LOCKED(s->tab, t) + { + lfjour_unregister(&s->digest_recipient); + rt_unlock_table(t); + } + ev_postpone(&s->update_event); - ASSERT_DIE(rt_export_get_state(&s->reader) == TES_DOWN); - - rfree(s->trie->lp); - rem_node(&s->roa_node); mb_free(s); - + channel_check_stopped(c); } @@ -1030,11 +965,6 @@ channel_config_new(const struct channel_class *cc, const char *name, uint net_ty cf->debug = new_config->channel_default_debug; cf->rpki_reload = 1; - cf->roa_settle = (struct settle_config) { - .min = 1 S, - .max = 20 S, - }; - add_tail(&proto->channels, &cf->n); return cf; @@ -1119,22 +1049,6 @@ channel_reconfigure(struct channel *c, struct channel_config *cf) c->in_req.trace_routes = c->out_req.trace_routes = c->debug | c->proto->debug; c->rpki_reload = cf->rpki_reload; - if ( (c->roa_settle.min != cf->roa_settle.min) - || (c->roa_settle.max != cf->roa_settle.max)) - { - c->roa_settle = cf->roa_settle; - - struct roa_subscription *s; - node *n; - - WALK_LIST2(s, n, c->roa_subscriptions, roa_node) - { - s->settle.cf = cf->roa_settle; - if (settle_active(&s->settle)) - settle_kick(&s->settle, &main_birdloop); - } - } - /* Execute channel-specific reconfigure hook */ if (c->class->reconfigure && !c->class->reconfigure(c, cf, &import_changed, &export_changed)) return 0; @@ -2608,7 +2522,7 @@ channel_create_reload_request(struct proto_reload_request *prr) { if (!prr->trie) return NULL; - + /* Increase the refeed counter */ atomic_fetch_add_explicit(&prr->counter, 1, memory_order_relaxed); ASSERT_DIE(this_cli->parser_pool != prr->trie->lp); diff --git a/nest/protocol.h b/nest/protocol.h index cce61cb8..ed36f418 100644 --- a/nest/protocol.h +++ b/nest/protocol.h @@ -511,8 +511,6 @@ struct channel_config { struct channel_limit in_limit; /* Limit for importing routes from protocol */ struct channel_limit out_limit; /* Limit for exporting routes to protocol */ - struct settle_config roa_settle; /* Settle times for ROA-induced reload */ - u8 net_type; /* Routing table network type (NET_*), 0 for undefined */ u8 ra_mode; /* Mode of received route advertisements (RA_*) */ u16 preference; /* Default route preference */ @@ -541,8 +539,6 @@ struct channel { struct limit in_limit; /* Input limit */ struct limit out_limit; /* Output limit */ - struct settle_config roa_settle; /* Settle times for ROA-induced reload */ - u8 limit_actions[PLD_MAX]; /* Limit actions enum */ u8 limit_active; /* Flags for active limits */ diff --git a/nest/route.h b/nest/route.h index 1cc11f53..6c1e9da5 100644 --- a/nest/route.h +++ b/nest/route.h @@ -70,6 +70,8 @@ struct rtable_config { struct settle_config export_settle; /* Export announcement settler */ struct settle_config export_rr_settle;/* Export announcement settler config valid when any route refresh is running */ + struct settle_config roa_settle; /* Settle times for ROA-induced reload */ + }; /* @@ -194,7 +196,7 @@ struct rt_export_union { } *update; const struct rt_export_feed { uint count_routes, count_exports; - const struct netindex *ni; + struct netindex *ni; rte *block; u64 *exports; char data[0]; @@ -213,7 +215,7 @@ struct rt_exporter { netindex_hash *netindex; /* Table for net <-> id conversion */ void (*stopped)(struct rt_exporter *); /* Callback when exporter can stop */ void (*cleanup_done)(struct rt_exporter *, u64 end); /* Callback when cleanup has been done */ - struct rt_export_feed *(*feed_net)(struct rt_exporter *, struct rcu_unwinder *, const struct netindex *, const struct rt_export_item *first); + struct rt_export_feed *(*feed_net)(struct rt_exporter *, struct rcu_unwinder *, struct netindex *, const struct rt_export_item *first); void (*feed_cleanup)(struct rt_exporter *, struct rt_export_feeder *); }; @@ -404,6 +406,7 @@ struct rtable_private { struct tbf rl_pipe; /* Rate limiting token buffer for pipe collisions */ struct f_trie *flowspec_trie; /* Trie for evaluation of flowspec notifications */ + struct roa_digestor *roa_digest; /* Digest of changed ROAs export */ // struct mpls_domain *mpls_domain; /* Label allocator for MPLS */ }; @@ -640,6 +643,20 @@ struct hostcache { event source_event; }; +struct roa_digestor { + struct rt_export_request req; /* Notifier from the table */ + struct lfjour digest; /* Digest journal of struct roa_digest */ + struct settle settle; /* Settle timer before announcing digests */ + struct f_trie *trie; /* Trie to be announced */ + rtable *tab; /* Table this belongs to */ + event event; +}; + +struct roa_digest { + LFJOUR_ITEM_INHERIT(li); + struct f_trie *trie; /* Trie marking all prefixes where ROA have changed */ +}; + #define rte_update channel_rte_import /** * rte_update - enter a new update to a routing table diff --git a/nest/rt-table.c b/nest/rt-table.c index d9a0255e..8b7c16ac 100644 --- a/nest/rt-table.c +++ b/nest/rt-table.c @@ -2083,7 +2083,7 @@ rt_net_feed(rtable *t, const net_addr *a, const struct rt_pending_export *first) } static struct rt_export_feed * -rt_feed_net_all(struct rt_exporter *e, struct rcu_unwinder *u, const struct netindex *ni, const struct rt_export_item *_first) +rt_feed_net_all(struct rt_exporter *e, struct rcu_unwinder *u, struct netindex *ni, const struct rt_export_item *_first) { RT_READ_ANCHORED(SKIP_BACK(rtable, export_all, e), tr, u); return rt_net_feed_internal(tr, ni, SKIP_BACK(const struct rt_pending_export, it, _first)); @@ -2096,7 +2096,7 @@ rt_net_best(rtable *t, const net_addr *a) RT_READ(t, tr); - const struct netindex *i = net_find_index(t->netindex, a); + struct netindex *i = net_find_index(t->netindex, a); net *n = i ? net_find(tr, i) : NULL; if (!n) return rt; @@ -2111,7 +2111,7 @@ rt_net_best(rtable *t, const net_addr *a) } static struct rt_export_feed * -rt_feed_net_best(struct rt_exporter *e, struct rcu_unwinder *u, const struct netindex *ni, const struct rt_export_item *_first) +rt_feed_net_best(struct rt_exporter *e, struct rcu_unwinder *u, struct netindex *ni, const struct rt_export_item *_first) { SKIP_BACK_DECLARE(rtable, t, export_best, e); SKIP_BACK_DECLARE(const struct rt_pending_export, first, it, _first); @@ -2690,6 +2690,85 @@ rt_flowspec_reset_trie(struct rtable_private *tab) tab->flowspec_trie->ipv4 = ipv4; } +/* ROA digestor */ + +static void +rt_dump_roa_digestor_req(struct rt_export_request *req) +{ + debug(" ROA update digestor %s (%p)\n", req->name, req); +} + +static void +rt_cleanup_roa_digest(struct lfjour *j UNUSED, struct lfjour_item *i) +{ + SKIP_BACK_DECLARE(struct roa_digest, d, li, i); + rfree(d->trie->lp); +} + +static void +rt_roa_announce_digest(struct settle *s) +{ + SKIP_BACK_DECLARE(struct roa_digestor, d, settle, s); + + RT_LOCK(d->tab, tab); + + struct lfjour_item *it = lfjour_push_prepare(&d->digest); + if (it) + { + SKIP_BACK_DECLARE(struct roa_digest, dd, li, it); + dd->trie = d->trie; + lfjour_push_commit(&d->digest); + } + else + rfree(d->trie->lp); + + d->trie = f_new_trie(lp_new(tab->rp), 0); +} + +static void +rt_roa_update_net(struct roa_digestor *d, struct netindex *ni, uint maxlen) +{ + trie_add_prefix(d->trie, ni->addr, net_pxlen(ni->addr), maxlen); + settle_kick(&d->settle, d->tab->loop); +} + +static void +rt_roa_update(void *_d) +{ + struct roa_digestor *d = _d; + RT_LOCK(d->tab, tab); + + RT_EXPORT_WALK(&d->req, u) + { + struct netindex *ni = NULL; + switch (u->kind) + { + case RT_EXPORT_STOP: + bug("Main table export stopped"); + + case RT_EXPORT_FEED: + if (u->feed->count_routes) + ni = u->feed->ni; + break; + + case RT_EXPORT_UPDATE: + /* Only switched ROA from one source to another? No change indicated. */ + if (!u->update->new || !u->update->old) + ni = NET_TO_INDEX(u->update->new ? u->update->new->net : u->update->old->net); + break; + } + + if (ni) + rt_roa_update_net(d, ni, (tab->addr_type == NET_ROA6) ? 128 : 32); + + MAYBE_DEFER_TASK(birdloop_event_list(tab->loop), &d->event, + "ROA digestor update in %s", tab->name); + } +} + + +/* Routing table setup and free */ + static void rt_free(resource *_r) { @@ -2845,6 +2924,44 @@ rt_setup(pool *pp, struct rtable_config *cf) RT_EXPORT_WALK(&t->best_req, u) ASSERT_DIE(u->kind == RT_EXPORT_FEED); + /* Prepare the ROA digestor */ + if ((t->addr_type == NET_ROA6) || (t->addr_type == NET_ROA4)) + { + struct roa_digestor *d = mb_alloc(p, sizeof *d); + *d = (struct roa_digestor) { + .tab = RT_PUB(t), + .req = { + .name = mb_sprintf(p, "%s.roa-digestor", t->name), + .r = { + .target = birdloop_event_list(t->loop), + .event = &d->event, + }, + .pool = p, + .trace_routes = t->debug, + .dump = rt_dump_roa_digestor_req, + }, + .digest = { + .loop = t->loop, + .domain = t->lock.rtable, + .item_size = sizeof(struct roa_digest), + .item_done = rt_cleanup_roa_digest, + }, + .settle = SETTLE_INIT(&cf->roa_settle, rt_roa_announce_digest, NULL), + .event = { + .hook = rt_roa_update, + .data = d, + }, + .trie = f_new_trie(lp_new(t->rp), 0), + }; + + struct settle_config digest_settle_config = {}; + + rtex_export_subscribe(&t->export_best, &d->req); + lfjour_init(&d->digest, &digest_settle_config); + + t->roa_digest = d; + } + t->cork_threshold = cf->cork_threshold; t->rl_pipe = (struct tbf) TBF_DEFAULT_LOG_LIMITS; @@ -3894,6 +4011,10 @@ rt_new_table(struct symbol *s, uint addr_type) .min = 100 MS, .max = 3 S, }; + c->roa_settle = (struct settle_config) { + .min = 1 S, + .max = 20 S, + }; c->debug = new_config->table_default_debug; add_tail(&new_config->tables, &c->n); @@ -3943,6 +4064,14 @@ rt_shutdown(void *tab_) rtable *t = tab_; RT_LOCK(t, tab); + if (tab->roa_digest) + { + rtex_export_unsubscribe(&tab->roa_digest->req); + ASSERT_DIE(EMPTY_TLIST(lfjour_recipient, &tab->roa_digest->digest.recipients)); + ev_postpone(&tab->roa_digest->event); + settle_cancel(&tab->roa_digest->settle); + } + rtex_export_unsubscribe(&tab->best_req); rt_exporter_shutdown(&tab->export_best, NULL); @@ -4037,6 +4166,11 @@ rt_reconfigure(struct rtable_private *tab, struct rtable_config *new, struct rta if (new->cork_threshold.low != old->cork_threshold.low) rt_check_cork_low(tab); + if (tab->roa_digest && ( + (new->roa_settle.min != tab->roa_digest->settle.cf.min) + || (new->roa_settle.max != tab->roa_digest->settle.cf.max))) + tab->roa_digest->settle.cf = new->roa_settle; + return 1; } diff --git a/proto/bgp/attrs.c b/proto/bgp/attrs.c index 1d3bd793..b8b1c1e0 100644 --- a/proto/bgp/attrs.c +++ b/proto/bgp/attrs.c @@ -1945,7 +1945,7 @@ bgp_out_item_done(struct lfjour *j, struct lfjour_item *i) {} static struct rt_export_feed * -bgp_out_feed_net(struct rt_exporter *e, struct rcu_unwinder *u, const struct netindex *ni, const struct rt_export_item *_first) +bgp_out_feed_net(struct rt_exporter *e, struct rcu_unwinder *u, struct netindex *ni, const struct rt_export_item *_first) { struct rt_export_feed *feed = NULL; SKIP_BACK_DECLARE(struct bgp_channel, c, prefix_exporter, e);