Make the "not my pool" searching loop a tiny bit smarter, while

making the number of pools variable.  Do not document the malloc
conf settings atm, don't know yet if they will stay.  Thanks to all
the testers. ok deraadt@

2 files changed, 39 insertions, 22 deletions

diff --git a/src/lib/libc/include/thread_private.h b/src/lib/libc/include/thread_private.h
index ae8d554a8a..774e0cba1f 100644
--- a/src/lib/libc/include/thread_private.h
+++ b/src/lib/libc/include/thread_private.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: thread_private.h,v 1.33 2017/12/05 13:45:31 kettenis Exp $ */
+/* $OpenBSD: thread_private.h,v 1.34 2019/01/10 18:45:33 otto Exp $ */
 
 /* PUBLIC DOMAIN: No Rights Reserved. Marco S Hyman <marc@snafu.org> */
 
@@ -7,7 +7,7 @@
 
 #include <stdio.h>              /* for FILE and __isthreaded */
 
-#define _MALLOC_MUTEXES 4
+#define _MALLOC_MUTEXES 32
 void _malloc_init(int);
 #ifdef __LIBC__
 PROTO_NORMAL(_malloc_init);
diff --git a/src/lib/libc/stdlib/malloc.c b/src/lib/libc/stdlib/malloc.c
index de504cdd8b..2a1bcfc8b6 100644
--- a/src/lib/libc/stdlib/malloc.c
+++ b/src/lib/libc/stdlib/malloc.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: malloc.c,v 1.257 2018/12/10 07:57:49 otto Exp $       */
+/*      $OpenBSD: malloc.c,v 1.258 2019/01/10 18:45:33 otto Exp $       */
 /*
  * Copyright (c) 2008, 2010, 2011, 2016 Otto Moerbeek <otto@drijf.net>
  * Copyright (c) 2012 Matthew Dempsky <matthew@openbsd.org>
@@ -143,6 +143,8 @@ struct dir_info {
         size_t cheap_reallocs;
         size_t malloc_used;             /* bytes allocated */
         size_t malloc_guarded;          /* bytes used for guards */
+        size_t pool_searches;           /* searches for pool */
+        size_t other_pool;              /* searches in other pool */
 #define STATS_ADD(x,y)  ((x) += (y))
 #define STATS_SUB(x,y)  ((x) -= (y))
 #define STATS_INC(x)    ((x)++)
@@ -179,7 +181,9 @@ struct chunk_info {
 };
 
 struct malloc_readonly {
-        struct dir_info *malloc_pool[_MALLOC_MUTEXES];  /* Main bookkeeping information */
+                                        /* Main bookkeeping information */
+        struct dir_info *malloc_pool[_MALLOC_MUTEXES];
+        u_int   malloc_mutexes;         /* how much in actual use? */
         int     malloc_mt;              /* multi-threaded mode? */
         int     malloc_freecheck;       /* Extensive double free check */
         int     malloc_freeunmap;       /* mprotect free pages PROT_NONE? */
@@ -267,7 +271,7 @@ getpool(void)
                 return mopts.malloc_pool[0];
         else
                 return mopts.malloc_pool[TIB_GET()->tib_tid &
-                    (_MALLOC_MUTEXES - 1)];
+                    (mopts.malloc_mutexes - 1)];
 }
 
 static __dead void
@@ -316,6 +320,16 @@ static void
 omalloc_parseopt(char opt)
 {
         switch (opt) {
+        case '+':
+                mopts.malloc_mutexes <<= 1;
+                if (mopts.malloc_mutexes > _MALLOC_MUTEXES)
+                        mopts.malloc_mutexes = _MALLOC_MUTEXES;
+                break;
+        case '-':
+                mopts.malloc_mutexes >>= 1;
+                if (mopts.malloc_mutexes < 1)
+                        mopts.malloc_mutexes = 1;
+                break;
         case '>':
                 mopts.malloc_cache <<= 1;
                 if (mopts.malloc_cache > MALLOC_MAXCACHE)
@@ -395,6 +409,7 @@ omalloc_init(void)
         /*
          * Default options
          */
+        mopts.malloc_mutexes = 4;
         mopts.malloc_junk = 1;
         mopts.malloc_cache = MALLOC_DEFAULT_CACHE;
 
@@ -485,7 +500,7 @@ omalloc_poolinit(struct dir_info **dp)
                 for (j = 0; j < MALLOC_CHUNK_LISTS; j++)
                         LIST_INIT(&d->chunk_dir[i][j]);
         }
-        STATS_ADD(d->malloc_used, regioninfo_size);
+        STATS_ADD(d->malloc_used, regioninfo_size + 3 * MALLOC_PAGESIZE);
         d->canary1 = mopts.malloc_canary ^ (u_int32_t)(uintptr_t)d;
         d->canary2 = ~d->canary1;
 
@@ -1196,7 +1211,7 @@ _malloc_init(int from_rthreads)
         if (!mopts.malloc_canary)
                 omalloc_init();
 
-        max = from_rthreads ? _MALLOC_MUTEXES : 1;
+        max = from_rthreads ? mopts.malloc_mutexes : 1;
         if (((uintptr_t)&malloc_readonly & MALLOC_PAGEMASK) == 0)
                 mprotect(&malloc_readonly, sizeof(malloc_readonly),
                     PROT_READ | PROT_WRITE);
@@ -1281,16 +1296,19 @@ findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool,
         struct dir_info *pool = argpool;
         struct region_info *r = find(pool, p);
 
+        STATS_INC(pool->pool_searches);
         if (r == NULL) {
                 if (mopts.malloc_mt)  {
                         int i;
 
-                        for (i = 0; i < _MALLOC_MUTEXES; i++) {
-                                if (i == argpool->mutex)
-                                        continue;
+                        STATS_INC(pool->other_pool);
+                        for (i = 1; i < mopts.malloc_mutexes; i++) {
+                                int j = (argpool->mutex + i) &
+                                    (mopts.malloc_mutexes - 1);
+
                                 pool->active--;
                                 _MALLOC_UNLOCK(pool->mutex);
-                                pool = mopts.malloc_pool[i];
+                                pool = mopts.malloc_pool[j];
                                 _MALLOC_LOCK(pool->mutex);
                                 pool->active++;
                                 r = find(pool, p);
@@ -2220,14 +2238,13 @@ malloc_dump1(int fd, int poolno, struct dir_info *d)
                 return;
         dprintf(fd, "Region slots free %zu/%zu\n",
                 d->regions_free, d->regions_total);
-        dprintf(fd, "Finds %zu/%zu\n", d->finds,
-            d->find_collisions);
-        dprintf(fd, "Inserts %zu/%zu\n", d->inserts,
-            d->insert_collisions);
-        dprintf(fd, "Deletes %zu/%zu\n", d->deletes,
-            d->delete_moves);
+        dprintf(fd, "Finds %zu/%zu\n", d->finds, d->find_collisions);
+        dprintf(fd, "Inserts %zu/%zu\n", d->inserts, d->insert_collisions);
+        dprintf(fd, "Deletes %zu/%zu\n", d->deletes, d->delete_moves);
         dprintf(fd, "Cheap reallocs %zu/%zu\n",
             d->cheap_reallocs, d->cheap_realloc_tries);
+        dprintf(fd, "Other pool searches %zu/%zu\n",
+            d->other_pool, d->pool_searches);
         dprintf(fd, "In use %zu\n", d->malloc_used);
         dprintf(fd, "Guarded %zu\n", d->malloc_guarded);
         dump_free_chunk_info(fd, d);
@@ -2289,7 +2306,7 @@ malloc_gdump(int fd)
         int i;
         int saved_errno = errno;
 
-        for (i = 0; i < _MALLOC_MUTEXES; i++)
+        for (i = 0; i < mopts.malloc_mutexes; i++)
                 malloc_dump(fd, i, mopts.malloc_pool[i]);
 
         errno = saved_errno;
@@ -2305,15 +2322,15 @@ malloc_exit(void)
         if (fd != -1) {
                 dprintf(fd, "******** Start dump %s *******\n", __progname);
                 dprintf(fd,
-                    "MT=%d I=%d F=%d U=%d J=%d R=%d X=%d C=%d cache=%u G=%zu\n",
-                    mopts.malloc_mt, mopts.internal_funcs,
-                    mopts.malloc_freecheck,
+                    "MT=%d M=%u I=%d F=%d U=%d J=%d R=%d X=%d C=%d cache=%u G=%zu\n",
+                    mopts.malloc_mt, mopts.mallloc_mutexes,
+                    mopts.internal_funcs, mopts.malloc_freecheck,
                     mopts.malloc_freeunmap, mopts.malloc_junk,
                     mopts.malloc_realloc, mopts.malloc_xmalloc,
                     mopts.chunk_canaries, mopts.malloc_cache,
                     mopts.malloc_guard);
 
-                for (i = 0; i < _MALLOC_MUTEXES; i++)
+                for (i = 0; i < mopts.malloc_mutexes; i++)
                         malloc_dump(fd, i, mopts.malloc_pool[i]);
                 dprintf(fd, "******** End dump %s *******\n", __progname);
                 close(fd);