malloc: Add tcache path for calloc

This commit add tcache support in calloc() which can largely improve
the performance of small size allocation, especially in multi-thread
scenario. tcache_available() and tcache_try_malloc() are split out as
a helper function for better reusing the code.

Also fix tst-safe-linking failure after enabling tcache. In previous,
calloc() is used as a way to by-pass tcache in memory allocation and
trigger safe-linking check in fastbins path. With tcache enabled, it
needs extra workarounds to bypass tcache.

Result of bench-calloc-thread benchmark

Test Platform: Xeon-8380
Ratio: New / Original time_per_iteration (Lower is Better)

Threads#   | Ratio
-----------|------
1 thread   | 0.656
4 threads  | 0.470
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>

1 files changed, 68 insertions, 13 deletions

diff --git a/malloc/tst-safe-linking.c b/malloc/tst-safe-linking.c
index 01dd07004d..5302575ad1 100644
--- a/malloc/tst-safe-linking.c
+++ b/malloc/tst-safe-linking.c
@@ -111,22 +111,37 @@ test_fastbin (void *closure)
   int i;
   int mask = ((int *)closure)[0];
   size_t size = TCACHE_ALLOC_SIZE;
+  void * ps[TCACHE_FILL_COUNT];
+  void * pps[TCACHE_FILL_COUNT];
 
   printf ("++ fastbin ++\n");
 
+  /* Populate the fastbin list.  */
+  void * volatile a = calloc (1, size);
+  void * volatile b = calloc (1, size);
+  void * volatile c = calloc (1, size);
+  printf ("a=%p, b=%p, c=%p\n", a, b, c);
+
+  /* Chunks for later tcache filling from fastbins.  */
+  for (i = 0; i < TCACHE_FILL_COUNT; ++i)
+    {
+      void * volatile p = calloc (1, size);
+      pps[i] = p;
+    }
+
   /* Take the tcache out of the game.  */
   for (i = 0; i < TCACHE_FILL_COUNT; ++i)
     {
       void * volatile p = calloc (1, size);
-      printf ("p=%p\n", p);
-      free (p);
+      ps[i] = p;
     }
 
-  /* Populate the fastbin list.  */
-  void * volatile a = calloc (1, size);
-  void * volatile b = calloc (1, size);
-  void * volatile c = calloc (1, size);
-  printf ("a=%p, b=%p, c=%p\n", a, b, c);
+  for (i = 0; i < TCACHE_FILL_COUNT; ++i)
+    {
+      free (ps[i]);
+    }
+
+  /* Free abc will return to fastbin in FIFO order.  */
   free (a);
   free (b);
   free (c);
@@ -136,11 +151,43 @@ test_fastbin (void *closure)
   memset (c, mask & 0xFF, size);
   printf ("After: c=%p, c[0]=%p\n", c, ((void **)c)[0]);
 
+  /* Filling fastbins, will be copied to tcache later.  */
+  for (i = 0; i < TCACHE_FILL_COUNT; ++i)
+    {
+      free (pps[i]);
+    }
+
+  /* Drain out tcache to make sure later alloc from fastbins.  */
+  for (i = 0; i < TCACHE_FILL_COUNT; ++i)
+    {
+      void * volatile p = calloc (1, size);
+      ps[i] = p;
+    }
+
+  /* This line will also filling tcache with remain pps and c.  */
+  pps[TCACHE_FILL_COUNT - 1] = calloc (1, size);
+
+  /* Tcache is FILO, now the first one is c, take it out.  */
   c = calloc (1, size);
   printf ("Allocated: c=%p\n", c);
+
+  /* Drain out remain pps from tcache.  */
+  for (i = 0; i < TCACHE_FILL_COUNT - 1; ++i)
+    {
+      void * volatile p = calloc (1, size);
+      pps[i] = p;
+    }
+
   /* This line will trigger the Safe-Linking check.  */
   b = calloc (1, size);
   printf ("b=%p\n", b);
+
+  /* Free previous pointers. */
+  for (i = 0; i < TCACHE_FILL_COUNT; ++i)
+    {
+      free (ps[i]);
+      free (pps[i]);
+    }
 }
 
 /* Try corrupting the fastbin list and trigger a consolidate.  */
@@ -150,21 +197,29 @@ test_fastbin_consolidate (void *closure)
   int i;
   int mask = ((int*)closure)[0];
   size_t size = TCACHE_ALLOC_SIZE;
+  void * ps[TCACHE_FILL_COUNT];
 
   printf ("++ fastbin consolidate ++\n");
 
+  /* Populate the fastbin list.  */
+  void * volatile a = calloc (1, size);
+  void * volatile b = calloc (1, size);
+  void * volatile c = calloc (1, size);
+  printf ("a=%p, b=%p, c=%p\n", a, b, c);
+
   /* Take the tcache out of the game.  */
   for (i = 0; i < TCACHE_FILL_COUNT; ++i)
     {
       void * volatile p = calloc (1, size);
-      free (p);
+      ps[i] = p;
     }
 
-  /* Populate the fastbin list.  */
-  void * volatile a = calloc (1, size);
-  void * volatile b = calloc (1, size);
-  void * volatile c = calloc (1, size);
-  printf ("a=%p, b=%p, c=%p\n", a, b, c);
+  for (i = 0; i < TCACHE_FILL_COUNT; ++i)
+    {
+      free (ps[i]);
+    }
+
+  /* Free abc will return to fastbin.  */
   free (a);
   free (b);
   free (c);