From af73cc42157cc2e73d0cd92ec52cb4acbf9599da Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 26 Jun 2024 20:45:54 +0200 Subject: [PATCH] Allocator now reports cold pages --- conf/conf.h | 1 + lib/resource.h | 2 ++ nest/cmds.c | 9 ++++++- sysdep/unix/alloc.c | 60 +++++++++++++++++++++++++++----------------- sysdep/unix/config.Y | 3 ++- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/conf/conf.h b/conf/conf.h index 7ed22afc..9a06ad4d 100644 --- a/conf/conf.h +++ b/conf/conf.h @@ -51,6 +51,7 @@ struct config { DL_PING = 1, DL_WAKEUP = 2, DL_SCHEDULING = 4, + DL_ALLOCATOR = 8, DL_SOCKETS = 0x10, DL_EVENTS = 0x20, DL_TIMERS = 0x40, diff --git a/lib/resource.h b/lib/resource.h index 6b225c76..2d540723 100644 --- a/lib/resource.h +++ b/lib/resource.h @@ -148,6 +148,8 @@ void buffer_realloc(void **buf, unsigned *size, unsigned need, unsigned item_siz extern long page_size; extern _Atomic int pages_kept; extern _Atomic int pages_kept_locally; +extern _Atomic int pages_kept_cold; +extern _Atomic int pages_kept_cold_index; void *alloc_page(void); void free_page(void *); void flush_local_pages(void); diff --git a/nest/cmds.c b/nest/cmds.c index b3729a5b..b55d2c23 100644 --- a/nest/cmds.c +++ b/nest/cmds.c @@ -125,11 +125,18 @@ cmd_show_memory(void) struct resmem total = rmemsize(&root_pool); #ifdef HAVE_MMAP int pk = atomic_load_explicit(&pages_kept, memory_order_relaxed) - + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed); + + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed) + + atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed); print_size("Standby memory:", (struct resmem) { .overhead = page_size * pk }); total.overhead += page_size * pk; #endif + print_size("Total:", total); + +#ifdef HAVE_MMAP + struct size_args cold = get_size_args(atomic_load_explicit(&pages_kept_cold, memory_order_relaxed) * page_size); + cli_msg(-1018, "%-23s " SIZE_FORMAT, "Cold memory:", SIZE_ARGS(cold)); +#endif cli_msg(0, ""); } diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index ef383f36..02960956 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -12,6 +12,8 @@ #include "lib/event.h" #include "lib/io-loop.h" +#include "conf/conf.h" + #include #include #include @@ -122,6 +124,8 @@ long page_size = 0; static DOMAIN(resource) empty_pages_domain; static struct empty_pages *empty_pages = NULL; + _Atomic int pages_kept_cold = 0; + _Atomic int pages_kept_cold_index = 0; static struct free_page * _Atomic page_stack = NULL; static _Thread_local struct free_page * local_page_stack = NULL; @@ -160,6 +164,9 @@ long page_size = 0; # define use_fake 1 #endif +#define ALLOC_TRACE(fmt...) do { \ + if (atomic_load_explicit(&global_runtime, memory_order_relaxed)->latency_debug & DL_ALLOCATOR) log(L_TRACE "Allocator: " fmt, ##fmt); } while (0) + void * alloc_page(void) { @@ -219,6 +226,7 @@ alloc_page(void) PROTECT_PAGE(empty_pages); UNPROTECT_PAGE(fp); ajlog(fp, empty_pages, empty_pages->pos, AJT_ALLOC_COLD_STD); + atomic_fetch_sub_explicit(&pages_kept_cold, 1, memory_order_relaxed); } else { @@ -226,6 +234,9 @@ alloc_page(void) fp = (struct free_page *) empty_pages; empty_pages = empty_pages->next; ajlog(fp, empty_pages, 0, AJT_ALLOC_COLD_KEEPER); + atomic_fetch_sub_explicit(&pages_kept_cold_index, 1, memory_order_relaxed); + if (!empty_pages) + ALLOC_TRACE("Taken last page from cold storage"); } } UNLOCK_DOMAIN(resource, empty_pages_domain); @@ -339,25 +350,29 @@ page_cleanup(void *_ UNUSED) if (shutting_down) return; - ajlog(NULL, NULL, 0, AJT_CLEANUP_BEGIN); - - /* Prevent contention */ - struct free_page *stack = PAGE_STACK_GET; - - /* Always replace by zero */ - PAGE_STACK_PUT(NULL); - - if (!stack) - { - ajlog(NULL, NULL, 0, AJT_CLEANUP_NOTHING); + /* Pages allocated inbetween */ + uint pk = atomic_load_explicit(&pages_kept, memory_order_relaxed); + if (pk < KEEP_PAGES_MAX) return; - } + /* Walk the pages */ + ajlog(NULL, NULL, 0, AJT_CLEANUP_BEGIN); + uint count = 0; do { - struct free_page *fp = stack; - stack = atomic_load_explicit(&fp->next, memory_order_relaxed); + /* Get next hot page */ + struct free_page *fp = PAGE_STACK_GET; + if (!fp) { + PAGE_STACK_PUT(NULL); + ajlog(NULL, NULL, 0, AJT_CLEANUP_END); + return; + } + /* Reinstate the stack with the next page in list */ + PAGE_STACK_PUT(atomic_load_explicit(&fp->next, memory_order_relaxed)); + + /* Cold pages are locked */ LOCK_DOMAIN(resource, empty_pages_domain); + /* Empty pages are stored as pointers. To store them, we need a pointer block. */ if (!empty_pages || (empty_pages->pos == EP_POS_MAX)) { @@ -370,6 +385,7 @@ page_cleanup(void *_ UNUSED) PROTECT_PAGE(ep); empty_pages = ep; ajlog(empty_pages, empty_pages->next, 0, AJT_CLEANUP_COLD_KEEPER); + atomic_fetch_add_explicit(&pages_kept_cold_index, 1, memory_order_relaxed); } else { @@ -389,20 +405,18 @@ page_cleanup(void *_ UNUSED) ) < 0) bug("madvise(%p) failed: %m", fp); ajlog(fp, empty_pages, empty_pages->pos, AJT_CLEANUP_COLD_STD); + atomic_fetch_add_explicit(&pages_kept_cold, 1, memory_order_relaxed); } UNLOCK_DOMAIN(resource, empty_pages_domain); + count++; } - while ((atomic_fetch_sub_explicit(&pages_kept, 1, memory_order_relaxed) >= KEEP_PAGES_MAX / 2) && stack); + while (atomic_fetch_sub_explicit(&pages_kept, 1, memory_order_relaxed) >= KEEP_PAGES_MAX / 2); - while (stack) - { - struct free_page *f = stack; - stack = atomic_load_explicit(&f->next, memory_order_acquire); - UNPROTECT_PAGE(f); - free_page(f); + ALLOC_TRACE("Moved %u pages to cold storage, now %u cold, %u index", count, + atomic_load_explicit(&pages_kept_cold, memory_order_relaxed), + atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed) + ); - atomic_fetch_sub_explicit(&pages_kept, 1, memory_order_relaxed); - } ajlog(NULL, NULL, 0, AJT_CLEANUP_END); } #endif diff --git a/sysdep/unix/config.Y b/sysdep/unix/config.Y index 1b72007c..46365c85 100644 --- a/sysdep/unix/config.Y +++ b/sysdep/unix/config.Y @@ -19,7 +19,7 @@ CF_DECLS CF_KEYWORDS(LOG, SYSLOG, ALL, DEBUG, TRACE, INFO, REMOTE, WARNING, ERROR, AUTH, FATAL, BUG, STDERR, SOFT, UDP, PORT) CF_KEYWORDS(NAME, CONFIRM, UNDO, CHECK, TIMEOUT, DEBUG, LATENCY, LIMIT, WATCHDOG, WARNING, STATUS) -CF_KEYWORDS(PING, WAKEUP, SOCKETS, SCHEDULING, EVENTS, TIMERS) +CF_KEYWORDS(PING, WAKEUP, SOCKETS, SCHEDULING, EVENTS, TIMERS, ALLOCATOR) CF_KEYWORDS(GRACEFUL, RESTART, FIXED) %type log_mask log_mask_list log_cat cfg_timeout debug_unix latency_debug_mask latency_debug_flag latency_debug_list @@ -151,6 +151,7 @@ latency_debug_flag: | WAKEUP { $$ = DL_WAKEUP; } | SOCKETS { $$ = DL_SOCKETS; } | SCHEDULING { $$ = DL_SCHEDULING; } + | ALLOCATOR { $$ = DL_ALLOCATOR; } | EVENTS { $$ = DL_EVENTS; } | TIMERS { $$ = DL_TIMERS; } ;