From 42bac88a211a7fac9dd1bfe7d1e45e59ac50c24f Mon Sep 17 00:00:00 2001 From: Szabolcs Nagy Date: Wed, 27 Jan 2021 15:45:43 +0000 Subject: malloc: Use global flag instead of function pointer dispatch for mtag A flag check can be faster than function pointers because of how branch prediction and speculation works and it can also remove a layer of indirection when there is a mismatch between the malloc internal tag_* api and __libc_mtag_* target hooks. Memory tagging wrapper functions are moved to malloc.c from arena.c and the logic now checks mmap_enabled. The definition of tag_new_usable is moved after chunk related definitions. This refactoring also allows using mtag_enabled checks instead of USE_MTAG ifdefs when memory tagging support only changes code logic when memory tagging is enabled at runtime. Note: an "if (false)" code block is optimized away even at -O0 by gcc. Reviewed-by: DJ Delorie --- malloc/malloc.c | 58 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 20 deletions(-) (limited to 'malloc/malloc.c') diff --git a/malloc/malloc.c b/malloc/malloc.c index e8961bb1e8..9c3981febe 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -441,35 +441,41 @@ void *(*__morecore)(ptrdiff_t) = __default_morecore; */ #ifdef USE_MTAG +static bool mtag_enabled = false; +static int mtag_mmap_flags = 0; +static size_t mtag_granule_mask = ~(size_t)0; +#else +# define mtag_enabled false +# define mtag_mmap_flags 0 +#endif -/* Default implementaions when memory tagging is supported, but disabled. */ -static void * -__default_tag_region (void *ptr, size_t size) +static __always_inline void * +tag_region (void *ptr, size_t size) { + if (__glibc_unlikely (mtag_enabled)) + return __libc_mtag_tag_region (ptr, size); return ptr; } -static void * -__default_tag_nop (void *ptr) +static __always_inline void * +tag_new_memset (void *ptr, int val, size_t size) { - return ptr; + if (__glibc_unlikely (mtag_enabled)) + return __libc_mtag_memset_with_tag (__libc_mtag_new_tag (ptr), val, size); + return memset (ptr, val, size); } -static int mtag_mmap_flags = 0; -static size_t mtag_granule_mask = ~(size_t)0; - -static void *(*tag_new_memset)(void *, int, size_t) = memset; -static void *(*tag_region)(void *, size_t) = __default_tag_region; -static void *(*tag_new_usable)(void *) = __default_tag_nop; -static void *(*tag_at)(void *) = __default_tag_nop; +/* Defined later. */ +static void * +tag_new_usable (void *ptr); -#else -# define mtag_mmap_flags 0 -# define tag_new_memset(ptr, val, size) memset (ptr, val, size) -# define tag_region(ptr, size) (ptr) -# define tag_new_usable(ptr) (ptr) -# define tag_at(ptr) (ptr) -#endif +static __always_inline void * +tag_at (void *ptr) +{ + if (__glibc_unlikely (mtag_enabled)) + return __libc_mtag_address_get_tag (ptr); + return ptr; +} #include @@ -1460,6 +1466,18 @@ checked_request2size (size_t req, size_t *sz) __nonnull (1) #pragma GCC poison mchunk_size #pragma GCC poison mchunk_prev_size +static __always_inline void * +tag_new_usable (void *ptr) +{ + if (__glibc_unlikely (mtag_enabled) && ptr) + { + mchunkptr cp = mem2chunk(ptr); + ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), + CHUNK_AVAILABLE_SIZE (cp) - CHUNK_HDR_SZ); + } + return ptr; +} + /* -------------------- Internal data structures -------------------- -- cgit v1.2.3