summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorotto <>2016-03-14 17:20:26 +0000
committerotto <>2016-03-14 17:20:26 +0000
commit8daa7bac4b58e4f955388e48607a12538c45fc40 (patch)
tree8afc2be0c69ad29bf0132124608033ce8894d0a5
parent32c7768b61c95094702adc74eaa693e9f28c61a2 (diff)
downloadopenbsd-8daa7bac4b58e4f955388e48607a12538c45fc40.tar.gz
openbsd-8daa7bac4b58e4f955388e48607a12538c45fc40.tar.bz2
openbsd-8daa7bac4b58e4f955388e48607a12538c45fc40.zip
small step towards multiple pools: move two globls into the struct dir_info
ok @stefan armani@
-rw-r--r--src/lib/libc/stdlib/malloc.c238
1 files changed, 126 insertions, 112 deletions
diff --git a/src/lib/libc/stdlib/malloc.c b/src/lib/libc/stdlib/malloc.c
index 24b22854a5..0fa0d39b8f 100644
--- a/src/lib/libc/stdlib/malloc.c
+++ b/src/lib/libc/stdlib/malloc.c
@@ -1,4 +1,4 @@
1/* $OpenBSD: malloc.c,v 1.183 2016/03/13 18:34:21 guenther Exp $ */ 1/* $OpenBSD: malloc.c,v 1.184 2016/03/14 17:20:26 otto Exp $ */
2/* 2/*
3 * Copyright (c) 2008, 2010, 2011 Otto Moerbeek <otto@drijf.net> 3 * Copyright (c) 2008, 2010, 2011 Otto Moerbeek <otto@drijf.net>
4 * Copyright (c) 2012 Matthew Dempsky <matthew@openbsd.org> 4 * Copyright (c) 2012 Matthew Dempsky <matthew@openbsd.org>
@@ -93,13 +93,13 @@
93#define MQUERY(a, sz) mquery((a), (size_t)(sz), PROT_READ | PROT_WRITE, \ 93#define MQUERY(a, sz) mquery((a), (size_t)(sz), PROT_READ | PROT_WRITE, \
94 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, (off_t)0) 94 MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, (off_t)0)
95 95
96#define _MALLOC_LEAVE() if (__isthreaded) do { \ 96#define _MALLOC_LEAVE(d) if (__isthreaded) do { \
97 malloc_active--; \ 97 (d)->active--; \
98 _MALLOC_UNLOCK(); \ 98 _MALLOC_UNLOCK(); \
99} while (0) 99} while (0)
100#define _MALLOC_ENTER() if (__isthreaded) do { \ 100#define _MALLOC_ENTER(d) if (__isthreaded) do { \
101 _MALLOC_LOCK(); \ 101 _MALLOC_LOCK(); \
102 malloc_active++; \ 102 (d)->active++; \
103} while (0) 103} while (0)
104 104
105struct region_info { 105struct region_info {
@@ -114,6 +114,7 @@ LIST_HEAD(chunk_head, chunk_info);
114 114
115struct dir_info { 115struct dir_info {
116 u_int32_t canary1; 116 u_int32_t canary1;
117 int active; /* status of malloc */
117 struct region_info *r; /* region slots */ 118 struct region_info *r; /* region slots */
118 size_t regions_total; /* number of region slots */ 119 size_t regions_total; /* number of region slots */
119 size_t regions_free; /* number of free slots */ 120 size_t regions_free; /* number of free slots */
@@ -127,6 +128,7 @@ struct dir_info {
127 /* delayed free chunk slots */ 128 /* delayed free chunk slots */
128 void *delayed_chunks[MALLOC_DELAYED_CHUNK_MASK + 1]; 129 void *delayed_chunks[MALLOC_DELAYED_CHUNK_MASK + 1];
129 size_t rbytesused; /* random bytes used */ 130 size_t rbytesused; /* random bytes used */
131 char *func; /* current function */
130 u_char rbytes[32]; /* random bytes */ 132 u_char rbytes[32]; /* random bytes */
131 u_short chunk_start; 133 u_short chunk_start;
132#ifdef MALLOC_STATS 134#ifdef MALLOC_STATS
@@ -203,8 +205,6 @@ static union {
203#define getpool() mopts.malloc_pool 205#define getpool() mopts.malloc_pool
204 206
205char *malloc_options; /* compile-time options */ 207char *malloc_options; /* compile-time options */
206static char *malloc_func; /* current function */
207static int malloc_active; /* status of malloc */
208 208
209static u_char getrbyte(struct dir_info *d); 209static u_char getrbyte(struct dir_info *d);
210 210
@@ -241,7 +241,7 @@ hash(void *p)
241} 241}
242 242
243static void 243static void
244wrterror(char *msg, void *p) 244wrterror(struct dir_info *d, char *msg, void *p)
245{ 245{
246 char *q = " error: "; 246 char *q = " error: ";
247 struct iovec iov[7]; 247 struct iovec iov[7];
@@ -254,8 +254,13 @@ wrterror(char *msg, void *p)
254 iov[1].iov_base = pidbuf; 254 iov[1].iov_base = pidbuf;
255 snprintf(pidbuf, sizeof(pidbuf), "(%d) in ", getpid()); 255 snprintf(pidbuf, sizeof(pidbuf), "(%d) in ", getpid());
256 iov[1].iov_len = strlen(pidbuf); 256 iov[1].iov_len = strlen(pidbuf);
257 iov[2].iov_base = malloc_func; 257 if (d != NULL) {
258 iov[2].iov_len = strlen(malloc_func); 258 iov[2].iov_base = d->func;
259 iov[2].iov_len = strlen(d->func);
260 } else {
261 iov[2].iov_base = "unknown";
262 iov[2].iov_len = 7;
263 }
259 iov[3].iov_base = q; 264 iov[3].iov_base = q;
260 iov[3].iov_len = strlen(q); 265 iov[3].iov_len = strlen(q);
261 iov[4].iov_base = msg; 266 iov[4].iov_base = msg;
@@ -316,14 +321,14 @@ unmap(struct dir_info *d, void *p, size_t sz)
316 u_int i, offset; 321 u_int i, offset;
317 322
318 if (sz != PAGEROUND(sz)) { 323 if (sz != PAGEROUND(sz)) {
319 wrterror("munmap round", NULL); 324 wrterror(d, "munmap round", NULL);
320 return; 325 return;
321 } 326 }
322 327
323 if (psz > mopts.malloc_cache) { 328 if (psz > mopts.malloc_cache) {
324 i = munmap(p, sz); 329 i = munmap(p, sz);
325 if (i) 330 if (i)
326 wrterror("munmap", p); 331 wrterror(d, "munmap", p);
327 STATS_SUB(d->malloc_used, sz); 332 STATS_SUB(d->malloc_used, sz);
328 return; 333 return;
329 } 334 }
@@ -337,7 +342,7 @@ unmap(struct dir_info *d, void *p, size_t sz)
337 if (r->p != NULL) { 342 if (r->p != NULL) {
338 rsz = r->size << MALLOC_PAGESHIFT; 343 rsz = r->size << MALLOC_PAGESHIFT;
339 if (munmap(r->p, rsz)) 344 if (munmap(r->p, rsz))
340 wrterror("munmap", r->p); 345 wrterror(d, "munmap", r->p);
341 r->p = NULL; 346 r->p = NULL;
342 if (tounmap > r->size) 347 if (tounmap > r->size)
343 tounmap -= r->size; 348 tounmap -= r->size;
@@ -349,7 +354,7 @@ unmap(struct dir_info *d, void *p, size_t sz)
349 } 354 }
350 } 355 }
351 if (tounmap > 0) 356 if (tounmap > 0)
352 wrterror("malloc cache underflow", NULL); 357 wrterror(d, "malloc cache underflow", NULL);
353 for (i = 0; i < mopts.malloc_cache; i++) { 358 for (i = 0; i < mopts.malloc_cache; i++) {
354 r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)]; 359 r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)];
355 if (r->p == NULL) { 360 if (r->p == NULL) {
@@ -364,9 +369,9 @@ unmap(struct dir_info *d, void *p, size_t sz)
364 } 369 }
365 } 370 }
366 if (i == mopts.malloc_cache) 371 if (i == mopts.malloc_cache)
367 wrterror("malloc free slot lost", NULL); 372 wrterror(d, "malloc free slot lost", NULL);
368 if (d->free_regions_size > mopts.malloc_cache) 373 if (d->free_regions_size > mopts.malloc_cache)
369 wrterror("malloc cache overflow", NULL); 374 wrterror(d, "malloc cache overflow", NULL);
370} 375}
371 376
372static void 377static void
@@ -381,7 +386,7 @@ zapcacheregion(struct dir_info *d, void *p, size_t len)
381 if (r->p >= p && r->p <= (void *)((char *)p + len)) { 386 if (r->p >= p && r->p <= (void *)((char *)p + len)) {
382 rsz = r->size << MALLOC_PAGESHIFT; 387 rsz = r->size << MALLOC_PAGESHIFT;
383 if (munmap(r->p, rsz)) 388 if (munmap(r->p, rsz))
384 wrterror("munmap", r->p); 389 wrterror(d, "munmap", r->p);
385 r->p = NULL; 390 r->p = NULL;
386 d->free_regions_size -= r->size; 391 d->free_regions_size -= r->size;
387 r->size = 0; 392 r->size = 0;
@@ -400,15 +405,15 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
400 405
401 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) || 406 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
402 d->canary1 != ~d->canary2) 407 d->canary1 != ~d->canary2)
403 wrterror("internal struct corrupt", NULL); 408 wrterror(d, "internal struct corrupt", NULL);
404 if (sz != PAGEROUND(sz)) { 409 if (sz != PAGEROUND(sz)) {
405 wrterror("map round", NULL); 410 wrterror(d, "map round", NULL);
406 return MAP_FAILED; 411 return MAP_FAILED;
407 } 412 }
408 if (!hint && psz > d->free_regions_size) { 413 if (!hint && psz > d->free_regions_size) {
409 _MALLOC_LEAVE(); 414 _MALLOC_LEAVE(d);
410 p = MMAP(sz); 415 p = MMAP(sz);
411 _MALLOC_ENTER(); 416 _MALLOC_ENTER(d);
412 if (p != MAP_FAILED) 417 if (p != MAP_FAILED)
413 STATS_ADD(d->malloc_used, sz); 418 STATS_ADD(d->malloc_used, sz);
414 /* zero fill not needed */ 419 /* zero fill not needed */
@@ -458,10 +463,10 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
458 if (hint) 463 if (hint)
459 return MAP_FAILED; 464 return MAP_FAILED;
460 if (d->free_regions_size > mopts.malloc_cache) 465 if (d->free_regions_size > mopts.malloc_cache)
461 wrterror("malloc cache", NULL); 466 wrterror(d, "malloc cache", NULL);
462 _MALLOC_LEAVE(); 467 _MALLOC_LEAVE(d);
463 p = MMAP(sz); 468 p = MMAP(sz);
464 _MALLOC_ENTER(); 469 _MALLOC_ENTER(d);
465 if (p != MAP_FAILED) 470 if (p != MAP_FAILED)
466 STATS_ADD(d->malloc_used, sz); 471 STATS_ADD(d->malloc_used, sz);
467 /* zero fill not needed */ 472 /* zero fill not needed */
@@ -652,7 +657,7 @@ omalloc_init(struct dir_info **dp)
652 regioninfo_size = d->regions_total * sizeof(struct region_info); 657 regioninfo_size = d->regions_total * sizeof(struct region_info);
653 d->r = MMAP(regioninfo_size); 658 d->r = MMAP(regioninfo_size);
654 if (d->r == MAP_FAILED) { 659 if (d->r == MAP_FAILED) {
655 wrterror("malloc init mmap failed", NULL); 660 wrterror(NULL, "malloc init mmap failed", NULL);
656 d->regions_total = 0; 661 d->regions_total = 0;
657 return 1; 662 return 1;
658 } 663 }
@@ -715,7 +720,7 @@ omalloc_grow(struct dir_info *d)
715 } 720 }
716 /* avoid pages containing meta info to end up in cache */ 721 /* avoid pages containing meta info to end up in cache */
717 if (munmap(d->r, d->regions_total * sizeof(struct region_info))) 722 if (munmap(d->r, d->regions_total * sizeof(struct region_info)))
718 wrterror("munmap", d->r); 723 wrterror(d, "munmap", d->r);
719 else 724 else
720 STATS_SUB(d->malloc_used, 725 STATS_SUB(d->malloc_used,
721 d->regions_total * sizeof(struct region_info)); 726 d->regions_total * sizeof(struct region_info));
@@ -803,7 +808,7 @@ find(struct dir_info *d, void *p)
803 808
804 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) || 809 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
805 d->canary1 != ~d->canary2) 810 d->canary1 != ~d->canary2)
806 wrterror("internal struct corrupt", NULL); 811 wrterror(d, "internal struct corrupt", NULL);
807 p = MASK_POINTER(p); 812 p = MASK_POINTER(p);
808 index = hash(p) & mask; 813 index = hash(p) & mask;
809 r = d->r[index].p; 814 r = d->r[index].p;
@@ -826,9 +831,9 @@ delete(struct dir_info *d, struct region_info *ri)
826 size_t i, j, r; 831 size_t i, j, r;
827 832
828 if (d->regions_total & (d->regions_total - 1)) 833 if (d->regions_total & (d->regions_total - 1))
829 wrterror("regions_total not 2^x", NULL); 834 wrterror(d, "regions_total not 2^x", NULL);
830 d->regions_free++; 835 d->regions_free++;
831 STATS_INC(getpool()->deletes); 836 STATS_INC(d->deletes);
832 837
833 i = ri - d->r; 838 i = ri - d->r;
834 for (;;) { 839 for (;;) {
@@ -844,7 +849,7 @@ delete(struct dir_info *d, struct region_info *ri)
844 (j < i && i <= r)) 849 (j < i && i <= r))
845 continue; 850 continue;
846 d->r[j] = d->r[i]; 851 d->r[j] = d->r[i];
847 STATS_INC(getpool()->delete_moves); 852 STATS_INC(d->delete_moves);
848 break; 853 break;
849 } 854 }
850 855
@@ -910,7 +915,7 @@ omalloc_make_chunks(struct dir_info *d, int bits, int listnum)
910 915
911 bits++; 916 bits++;
912 if ((uintptr_t)pp & bits) 917 if ((uintptr_t)pp & bits)
913 wrterror("pp & bits", pp); 918 wrterror(d, "pp & bits", pp);
914 919
915 insert(d, (void *)((uintptr_t)pp | bits), (uintptr_t)bp, NULL); 920 insert(d, (void *)((uintptr_t)pp | bits), (uintptr_t)bp, NULL);
916 return bp; 921 return bp;
@@ -930,7 +935,7 @@ malloc_bytes(struct dir_info *d, size_t size, void *f)
930 935
931 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) || 936 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
932 d->canary1 != ~d->canary2) 937 d->canary1 != ~d->canary2)
933 wrterror("internal struct corrupt", NULL); 938 wrterror(d, "internal struct corrupt", NULL);
934 /* Don't bother with anything less than this */ 939 /* Don't bother with anything less than this */
935 /* unless we have a malloc(0) requests */ 940 /* unless we have a malloc(0) requests */
936 if (size != 0 && size < MALLOC_MINSIZE) 941 if (size != 0 && size < MALLOC_MINSIZE)
@@ -955,7 +960,7 @@ malloc_bytes(struct dir_info *d, size_t size, void *f)
955 } 960 }
956 961
957 if (bp->canary != d->canary1) 962 if (bp->canary != d->canary1)
958 wrterror("chunk info corrupted", NULL); 963 wrterror(d, "chunk info corrupted", NULL);
959 964
960 i = d->chunk_start; 965 i = d->chunk_start;
961 if (bp->free > 1) 966 if (bp->free > 1)
@@ -1018,25 +1023,25 @@ find_chunknum(struct dir_info *d, struct region_info *r, void *ptr)
1018 1023
1019 info = (struct chunk_info *)r->size; 1024 info = (struct chunk_info *)r->size;
1020 if (info->canary != d->canary1) 1025 if (info->canary != d->canary1)
1021 wrterror("chunk info corrupted", NULL); 1026 wrterror(d, "chunk info corrupted", NULL);
1022 1027
1023 if (mopts.malloc_canaries && info->size > 0) { 1028 if (mopts.malloc_canaries && info->size > 0) {
1024 char *end = (char *)ptr + info->size; 1029 char *end = (char *)ptr + info->size;
1025 uintptr_t *canary = (uintptr_t *)(end - mopts.malloc_canaries); 1030 uintptr_t *canary = (uintptr_t *)(end - mopts.malloc_canaries);
1026 if (*canary != (mopts.malloc_chunk_canary ^ hash(canary))) 1031 if (*canary != (mopts.malloc_chunk_canary ^ hash(canary)))
1027 wrterror("chunk canary corrupted", ptr); 1032 wrterror(d, "chunk canary corrupted", ptr);
1028 } 1033 }
1029 1034
1030 /* Find the chunk number on the page */ 1035 /* Find the chunk number on the page */
1031 chunknum = ((uintptr_t)ptr & MALLOC_PAGEMASK) >> info->shift; 1036 chunknum = ((uintptr_t)ptr & MALLOC_PAGEMASK) >> info->shift;
1032 1037
1033 if ((uintptr_t)ptr & ((1U << (info->shift)) - 1)) { 1038 if ((uintptr_t)ptr & ((1U << (info->shift)) - 1)) {
1034 wrterror("modified chunk-pointer", ptr); 1039 wrterror(d, "modified chunk-pointer", ptr);
1035 return -1; 1040 return -1;
1036 } 1041 }
1037 if (info->bits[chunknum / MALLOC_BITS] & 1042 if (info->bits[chunknum / MALLOC_BITS] &
1038 (1U << (chunknum % MALLOC_BITS))) { 1043 (1U << (chunknum % MALLOC_BITS))) {
1039 wrterror("chunk is already free", ptr); 1044 wrterror(d, "chunk is already free", ptr);
1040 return -1; 1045 return -1;
1041 } 1046 }
1042 return chunknum; 1047 return chunknum;
@@ -1092,9 +1097,8 @@ free_bytes(struct dir_info *d, struct region_info *r, void *ptr)
1092 1097
1093 1098
1094static void * 1099static void *
1095omalloc(size_t sz, int zero_fill, void *f) 1100omalloc(struct dir_info *pool, size_t sz, int zero_fill, void *f)
1096{ 1101{
1097 struct dir_info *pool = getpool();
1098 void *p; 1102 void *p;
1099 size_t psz; 1103 size_t psz;
1100 1104
@@ -1118,7 +1122,7 @@ omalloc(size_t sz, int zero_fill, void *f)
1118 if (mopts.malloc_guard) { 1122 if (mopts.malloc_guard) {
1119 if (mprotect((char *)p + psz - mopts.malloc_guard, 1123 if (mprotect((char *)p + psz - mopts.malloc_guard,
1120 mopts.malloc_guard, PROT_NONE)) 1124 mopts.malloc_guard, PROT_NONE))
1121 wrterror("mprotect", NULL); 1125 wrterror(pool, "mprotect", NULL);
1122 STATS_ADD(pool->malloc_guarded, mopts.malloc_guard); 1126 STATS_ADD(pool->malloc_guarded, mopts.malloc_guard);
1123 } 1127 }
1124 1128
@@ -1161,15 +1165,15 @@ omalloc(size_t sz, int zero_fill, void *f)
1161 * potentially worse. 1165 * potentially worse.
1162 */ 1166 */
1163static void 1167static void
1164malloc_recurse(void) 1168malloc_recurse(struct dir_info *d)
1165{ 1169{
1166 static int noprint; 1170 static int noprint;
1167 1171
1168 if (noprint == 0) { 1172 if (noprint == 0) {
1169 noprint = 1; 1173 noprint = 1;
1170 wrterror("recursive call", NULL); 1174 wrterror(d, "recursive call", NULL);
1171 } 1175 }
1172 malloc_active--; 1176 d->active--;
1173 _MALLOC_UNLOCK(); 1177 _MALLOC_UNLOCK();
1174 errno = EDEADLK; 1178 errno = EDEADLK;
1175} 1179}
@@ -1180,7 +1184,7 @@ malloc_init(void)
1180 if (omalloc_init(&mopts.malloc_pool)) { 1184 if (omalloc_init(&mopts.malloc_pool)) {
1181 _MALLOC_UNLOCK(); 1185 _MALLOC_UNLOCK();
1182 if (mopts.malloc_xmalloc) 1186 if (mopts.malloc_xmalloc)
1183 wrterror("out of memory", NULL); 1187 wrterror(NULL, "out of memory", NULL);
1184 errno = ENOMEM; 1188 errno = ENOMEM;
1185 return -1; 1189 return -1;
1186 } 1190 }
@@ -1191,26 +1195,29 @@ void *
1191malloc(size_t size) 1195malloc(size_t size)
1192{ 1196{
1193 void *r; 1197 void *r;
1198 struct dir_info *d;
1194 int saved_errno = errno; 1199 int saved_errno = errno;
1195 1200
1196 _MALLOC_LOCK(); 1201 _MALLOC_LOCK();
1197 malloc_func = "malloc():"; 1202 d = getpool();
1198 if (getpool() == NULL) { 1203 if (d == NULL) {
1199 if (malloc_init() != 0) 1204 if (malloc_init() != 0)
1200 return NULL; 1205 return NULL;
1206 d = getpool();
1201 } 1207 }
1208 d->func = "malloc():";
1202 1209
1203 if (malloc_active++) { 1210 if (d->active++) {
1204 malloc_recurse(); 1211 malloc_recurse(d);
1205 return NULL; 1212 return NULL;
1206 } 1213 }
1207 if (size > 0 && size <= MALLOC_MAXCHUNK) 1214 if (size > 0 && size <= MALLOC_MAXCHUNK)
1208 size += mopts.malloc_canaries; 1215 size += mopts.malloc_canaries;
1209 r = omalloc(size, 0, CALLER); 1216 r = omalloc(d, size, 0, CALLER);
1210 malloc_active--; 1217 d->active--;
1211 _MALLOC_UNLOCK(); 1218 _MALLOC_UNLOCK();
1212 if (r == NULL && mopts.malloc_xmalloc) { 1219 if (r == NULL && mopts.malloc_xmalloc) {
1213 wrterror("out of memory", NULL); 1220 wrterror(d, "out of memory", NULL);
1214 errno = ENOMEM; 1221 errno = ENOMEM;
1215 } 1222 }
1216 if (r != NULL) 1223 if (r != NULL)
@@ -1220,16 +1227,15 @@ malloc(size_t size)
1220/*DEF_STRONG(malloc);*/ 1227/*DEF_STRONG(malloc);*/
1221 1228
1222static void 1229static void
1223validate_junk(void *p) { 1230validate_junk(struct dir_info *pool, void *p) {
1224 struct region_info *r; 1231 struct region_info *r;
1225 struct dir_info *pool = getpool();
1226 size_t byte, sz; 1232 size_t byte, sz;
1227 1233
1228 if (p == NULL) 1234 if (p == NULL)
1229 return; 1235 return;
1230 r = find(pool, p); 1236 r = find(pool, p);
1231 if (r == NULL) { 1237 if (r == NULL) {
1232 wrterror("bogus pointer in validate_junk", p); 1238 wrterror(pool, "bogus pointer in validate_junk", p);
1233 return; 1239 return;
1234 } 1240 }
1235 REALSIZE(sz, r); 1241 REALSIZE(sz, r);
@@ -1239,22 +1245,21 @@ validate_junk(void *p) {
1239 sz = 32; 1245 sz = 32;
1240 for (byte = 0; byte < sz; byte++) { 1246 for (byte = 0; byte < sz; byte++) {
1241 if (((unsigned char *)p)[byte] != SOME_FREEJUNK) { 1247 if (((unsigned char *)p)[byte] != SOME_FREEJUNK) {
1242 wrterror("use after free", p); 1248 wrterror(pool, "use after free", p);
1243 return; 1249 return;
1244 } 1250 }
1245 } 1251 }
1246} 1252}
1247 1253
1248static void 1254static void
1249ofree(void *p) 1255ofree(struct dir_info *pool, void *p)
1250{ 1256{
1251 struct dir_info *pool = getpool();
1252 struct region_info *r; 1257 struct region_info *r;
1253 size_t sz; 1258 size_t sz;
1254 1259
1255 r = find(pool, p); 1260 r = find(pool, p);
1256 if (r == NULL) { 1261 if (r == NULL) {
1257 wrterror("bogus pointer (double free?)", p); 1262 wrterror(pool, "bogus pointer (double free?)", p);
1258 return; 1263 return;
1259 } 1264 }
1260 REALSIZE(sz, r); 1265 REALSIZE(sz, r);
@@ -1262,7 +1267,7 @@ ofree(void *p)
1262 if (sz - mopts.malloc_guard >= MALLOC_PAGESIZE - 1267 if (sz - mopts.malloc_guard >= MALLOC_PAGESIZE -
1263 MALLOC_LEEWAY) { 1268 MALLOC_LEEWAY) {
1264 if (r->p != p) { 1269 if (r->p != p) {
1265 wrterror("bogus pointer", p); 1270 wrterror(pool, "bogus pointer", p);
1266 return; 1271 return;
1267 } 1272 }
1268 } else { 1273 } else {
@@ -1277,12 +1282,12 @@ ofree(void *p)
1277 } 1282 }
1278 if (mopts.malloc_guard) { 1283 if (mopts.malloc_guard) {
1279 if (sz < mopts.malloc_guard) 1284 if (sz < mopts.malloc_guard)
1280 wrterror("guard size", NULL); 1285 wrterror(pool, "guard size", NULL);
1281 if (!mopts.malloc_freeunmap) { 1286 if (!mopts.malloc_freeunmap) {
1282 if (mprotect((char *)p + PAGEROUND(sz) - 1287 if (mprotect((char *)p + PAGEROUND(sz) -
1283 mopts.malloc_guard, mopts.malloc_guard, 1288 mopts.malloc_guard, mopts.malloc_guard,
1284 PROT_READ | PROT_WRITE)) 1289 PROT_READ | PROT_WRITE))
1285 wrterror("mprotect", NULL); 1290 wrterror(pool, "mprotect", NULL);
1286 } 1291 }
1287 STATS_SUB(pool->malloc_guarded, mopts.malloc_guard); 1292 STATS_SUB(pool->malloc_guarded, mopts.malloc_guard);
1288 } 1293 }
@@ -1306,17 +1311,17 @@ ofree(void *p)
1306 tmp = p; 1311 tmp = p;
1307 p = pool->delayed_chunks[i]; 1312 p = pool->delayed_chunks[i];
1308 if (tmp == p) { 1313 if (tmp == p) {
1309 wrterror("double free", p); 1314 wrterror(pool, "double free", p);
1310 return; 1315 return;
1311 } 1316 }
1312 if (mopts.malloc_junk) 1317 if (mopts.malloc_junk)
1313 validate_junk(p); 1318 validate_junk(pool, p);
1314 pool->delayed_chunks[i] = tmp; 1319 pool->delayed_chunks[i] = tmp;
1315 } 1320 }
1316 if (p != NULL) { 1321 if (p != NULL) {
1317 r = find(pool, p); 1322 r = find(pool, p);
1318 if (r == NULL) { 1323 if (r == NULL) {
1319 wrterror("bogus pointer (double free?)", p); 1324 wrterror(pool, "bogus pointer (double free?)", p);
1320 return; 1325 return;
1321 } 1326 }
1322 free_bytes(pool, r, p); 1327 free_bytes(pool, r, p);
@@ -1327,6 +1332,7 @@ ofree(void *p)
1327void 1332void
1328free(void *ptr) 1333free(void *ptr)
1329{ 1334{
1335 struct dir_info *d;
1330 int saved_errno = errno; 1336 int saved_errno = errno;
1331 1337
1332 /* This is legal. */ 1338 /* This is legal. */
@@ -1334,18 +1340,19 @@ free(void *ptr)
1334 return; 1340 return;
1335 1341
1336 _MALLOC_LOCK(); 1342 _MALLOC_LOCK();
1337 malloc_func = "free():"; 1343 d = getpool();
1338 if (getpool() == NULL) { 1344 if (d == NULL) {
1339 _MALLOC_UNLOCK(); 1345 _MALLOC_UNLOCK();
1340 wrterror("free() called before allocation", NULL); 1346 wrterror(d, "free() called before allocation", NULL);
1341 return; 1347 return;
1342 } 1348 }
1343 if (malloc_active++) { 1349 d->func = "free():";
1344 malloc_recurse(); 1350 if (d->active++) {
1351 malloc_recurse(d);
1345 return; 1352 return;
1346 } 1353 }
1347 ofree(ptr); 1354 ofree(d, ptr);
1348 malloc_active--; 1355 d->active--;
1349 _MALLOC_UNLOCK(); 1356 _MALLOC_UNLOCK();
1350 errno = saved_errno; 1357 errno = saved_errno;
1351} 1358}
@@ -1353,19 +1360,18 @@ free(void *ptr)
1353 1360
1354 1361
1355static void * 1362static void *
1356orealloc(void *p, size_t newsz, void *f) 1363orealloc(struct dir_info *pool, void *p, size_t newsz, void *f)
1357{ 1364{
1358 struct dir_info *pool = getpool();
1359 struct region_info *r; 1365 struct region_info *r;
1360 size_t oldsz, goldsz, gnewsz; 1366 size_t oldsz, goldsz, gnewsz;
1361 void *q; 1367 void *q;
1362 1368
1363 if (p == NULL) 1369 if (p == NULL)
1364 return omalloc(newsz, 0, f); 1370 return omalloc(pool, newsz, 0, f);
1365 1371
1366 r = find(pool, p); 1372 r = find(pool, p);
1367 if (r == NULL) { 1373 if (r == NULL) {
1368 wrterror("bogus pointer (double free?)", p); 1374 wrterror(pool, "bogus pointer (double free?)", p);
1369 return NULL; 1375 return NULL;
1370 } 1376 }
1371 if (newsz >= SIZE_MAX - mopts.malloc_guard - MALLOC_PAGESIZE) { 1377 if (newsz >= SIZE_MAX - mopts.malloc_guard - MALLOC_PAGESIZE) {
@@ -1377,7 +1383,7 @@ orealloc(void *p, size_t newsz, void *f)
1377 goldsz = oldsz; 1383 goldsz = oldsz;
1378 if (oldsz > MALLOC_MAXCHUNK) { 1384 if (oldsz > MALLOC_MAXCHUNK) {
1379 if (oldsz < mopts.malloc_guard) 1385 if (oldsz < mopts.malloc_guard)
1380 wrterror("guard size", NULL); 1386 wrterror(pool, "guard size", NULL);
1381 oldsz -= mopts.malloc_guard; 1387 oldsz -= mopts.malloc_guard;
1382 } 1388 }
1383 1389
@@ -1416,7 +1422,7 @@ gotit:
1416 return p; 1422 return p;
1417 } else if (q != MAP_FAILED) { 1423 } else if (q != MAP_FAILED) {
1418 if (munmap(q, needed)) 1424 if (munmap(q, needed))
1419 wrterror("munmap", q); 1425 wrterror(pool, "munmap", q);
1420 } 1426 }
1421 } 1427 }
1422 } else if (rnewsz < roldsz) { 1428 } else if (rnewsz < roldsz) {
@@ -1424,11 +1430,11 @@ gotit:
1424 if (mprotect((char *)p + roldsz - 1430 if (mprotect((char *)p + roldsz -
1425 mopts.malloc_guard, mopts.malloc_guard, 1431 mopts.malloc_guard, mopts.malloc_guard,
1426 PROT_READ | PROT_WRITE)) 1432 PROT_READ | PROT_WRITE))
1427 wrterror("mprotect", NULL); 1433 wrterror(pool, "mprotect", NULL);
1428 if (mprotect((char *)p + rnewsz - 1434 if (mprotect((char *)p + rnewsz -
1429 mopts.malloc_guard, mopts.malloc_guard, 1435 mopts.malloc_guard, mopts.malloc_guard,
1430 PROT_NONE)) 1436 PROT_NONE))
1431 wrterror("mprotect", NULL); 1437 wrterror(pool, "mprotect", NULL);
1432 } 1438 }
1433 unmap(pool, (char *)p + rnewsz, roldsz - rnewsz); 1439 unmap(pool, (char *)p + rnewsz, roldsz - rnewsz);
1434 r->size = gnewsz; 1440 r->size = gnewsz;
@@ -1454,7 +1460,7 @@ gotit:
1454 STATS_SETF(r, f); 1460 STATS_SETF(r, f);
1455 return p; 1461 return p;
1456 } else if (newsz != oldsz || mopts.malloc_realloc) { 1462 } else if (newsz != oldsz || mopts.malloc_realloc) {
1457 q = omalloc(newsz, 0, f); 1463 q = omalloc(pool, newsz, 0, f);
1458 if (q == NULL) 1464 if (q == NULL)
1459 return NULL; 1465 return NULL;
1460 if (newsz != 0 && oldsz != 0) { 1466 if (newsz != 0 && oldsz != 0) {
@@ -1463,7 +1469,7 @@ gotit:
1463 copysz -= mopts.malloc_canaries; 1469 copysz -= mopts.malloc_canaries;
1464 memcpy(q, p, copysz); 1470 memcpy(q, p, copysz);
1465 } 1471 }
1466 ofree(p); 1472 ofree(pool, p);
1467 return q; 1473 return q;
1468 } else { 1474 } else {
1469 STATS_SETF(r, f); 1475 STATS_SETF(r, f);
@@ -1474,27 +1480,30 @@ gotit:
1474void * 1480void *
1475realloc(void *ptr, size_t size) 1481realloc(void *ptr, size_t size)
1476{ 1482{
1483 struct dir_info *d;
1477 void *r; 1484 void *r;
1478 int saved_errno = errno; 1485 int saved_errno = errno;
1479 1486
1480 _MALLOC_LOCK(); 1487 _MALLOC_LOCK();
1481 malloc_func = "realloc():"; 1488 d = getpool();
1482 if (getpool() == NULL) { 1489 if (d == NULL) {
1483 if (malloc_init() != 0) 1490 if (malloc_init() != 0)
1484 return NULL; 1491 return NULL;
1492 d = getpool();
1485 } 1493 }
1486 if (malloc_active++) { 1494 d->func = "realloc():";
1487 malloc_recurse(); 1495 if (d->active++) {
1496 malloc_recurse(d);
1488 return NULL; 1497 return NULL;
1489 } 1498 }
1490 if (size > 0 && size <= MALLOC_MAXCHUNK) 1499 if (size > 0 && size <= MALLOC_MAXCHUNK)
1491 size += mopts.malloc_canaries; 1500 size += mopts.malloc_canaries;
1492 r = orealloc(ptr, size, CALLER); 1501 r = orealloc(d, ptr, size, CALLER);
1493 1502
1494 malloc_active--; 1503 d->active--;
1495 _MALLOC_UNLOCK(); 1504 _MALLOC_UNLOCK();
1496 if (r == NULL && mopts.malloc_xmalloc) { 1505 if (r == NULL && mopts.malloc_xmalloc) {
1497 wrterror("out of memory", NULL); 1506 wrterror(d, "out of memory", NULL);
1498 errno = ENOMEM; 1507 errno = ENOMEM;
1499 } 1508 }
1500 if (r != NULL) 1509 if (r != NULL)
@@ -1513,38 +1522,41 @@ realloc(void *ptr, size_t size)
1513void * 1522void *
1514calloc(size_t nmemb, size_t size) 1523calloc(size_t nmemb, size_t size)
1515{ 1524{
1525 struct dir_info *d;
1516 void *r; 1526 void *r;
1517 int saved_errno = errno; 1527 int saved_errno = errno;
1518 1528
1519 _MALLOC_LOCK(); 1529 _MALLOC_LOCK();
1520 malloc_func = "calloc():"; 1530 d = getpool();
1521 if (getpool() == NULL) { 1531 if (d == NULL) {
1522 if (malloc_init() != 0) 1532 if (malloc_init() != 0)
1523 return NULL; 1533 return NULL;
1534 d = getpool();
1524 } 1535 }
1536 d->func = "calloc():";
1525 if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) && 1537 if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
1526 nmemb > 0 && SIZE_MAX / nmemb < size) { 1538 nmemb > 0 && SIZE_MAX / nmemb < size) {
1527 _MALLOC_UNLOCK(); 1539 _MALLOC_UNLOCK();
1528 if (mopts.malloc_xmalloc) 1540 if (mopts.malloc_xmalloc)
1529 wrterror("out of memory", NULL); 1541 wrterror(d, "out of memory", NULL);
1530 errno = ENOMEM; 1542 errno = ENOMEM;
1531 return NULL; 1543 return NULL;
1532 } 1544 }
1533 1545
1534 if (malloc_active++) { 1546 if (d->active++) {
1535 malloc_recurse(); 1547 malloc_recurse(d);
1536 return NULL; 1548 return NULL;
1537 } 1549 }
1538 1550
1539 size *= nmemb; 1551 size *= nmemb;
1540 if (size > 0 && size <= MALLOC_MAXCHUNK) 1552 if (size > 0 && size <= MALLOC_MAXCHUNK)
1541 size += mopts.malloc_canaries; 1553 size += mopts.malloc_canaries;
1542 r = omalloc(size, 1, CALLER); 1554 r = omalloc(d, size, 1, CALLER);
1543 1555
1544 malloc_active--; 1556 d->active--;
1545 _MALLOC_UNLOCK(); 1557 _MALLOC_UNLOCK();
1546 if (r == NULL && mopts.malloc_xmalloc) { 1558 if (r == NULL && mopts.malloc_xmalloc) {
1547 wrterror("out of memory", NULL); 1559 wrterror(d, "out of memory", NULL);
1548 errno = ENOMEM; 1560 errno = ENOMEM;
1549 } 1561 }
1550 if (r != NULL) 1562 if (r != NULL)
@@ -1559,11 +1571,11 @@ mapalign(struct dir_info *d, size_t alignment, size_t sz, int zero_fill)
1559 char *p, *q; 1571 char *p, *q;
1560 1572
1561 if (alignment < MALLOC_PAGESIZE || ((alignment - 1) & alignment) != 0) { 1573 if (alignment < MALLOC_PAGESIZE || ((alignment - 1) & alignment) != 0) {
1562 wrterror("mapalign bad alignment", NULL); 1574 wrterror(d, "mapalign bad alignment", NULL);
1563 return MAP_FAILED; 1575 return MAP_FAILED;
1564 } 1576 }
1565 if (sz != PAGEROUND(sz)) { 1577 if (sz != PAGEROUND(sz)) {
1566 wrterror("mapalign round", NULL); 1578 wrterror(d, "mapalign round", NULL);
1567 return MAP_FAILED; 1579 return MAP_FAILED;
1568 } 1580 }
1569 1581
@@ -1582,19 +1594,18 @@ mapalign(struct dir_info *d, size_t alignment, size_t sz, int zero_fill)
1582 q = (char *)(((uintptr_t)p + alignment - 1) & ~(alignment - 1)); 1594 q = (char *)(((uintptr_t)p + alignment - 1) & ~(alignment - 1));
1583 if (q != p) { 1595 if (q != p) {
1584 if (munmap(p, q - p)) 1596 if (munmap(p, q - p))
1585 wrterror("munmap", p); 1597 wrterror(d, "munmap", p);
1586 } 1598 }
1587 if (munmap(q + sz, alignment - (q - p))) 1599 if (munmap(q + sz, alignment - (q - p)))
1588 wrterror("munmap", q + sz); 1600 wrterror(d, "munmap", q + sz);
1589 STATS_SUB(d->malloc_used, alignment); 1601 STATS_SUB(d->malloc_used, alignment);
1590 1602
1591 return q; 1603 return q;
1592} 1604}
1593 1605
1594static void * 1606static void *
1595omemalign(size_t alignment, size_t sz, int zero_fill, void *f) 1607omemalign(struct dir_info *pool, size_t alignment, size_t sz, int zero_fill, void *f)
1596{ 1608{
1597 struct dir_info *pool = getpool();
1598 size_t psz; 1609 size_t psz;
1599 void *p; 1610 void *p;
1600 1611
@@ -1605,7 +1616,7 @@ omemalign(size_t alignment, size_t sz, int zero_fill, void *f)
1605 */ 1616 */
1606 if (sz < alignment) 1617 if (sz < alignment)
1607 sz = alignment; 1618 sz = alignment;
1608 return omalloc(sz, zero_fill, f); 1619 return omalloc(pool, sz, zero_fill, f);
1609 } 1620 }
1610 1621
1611 if (sz >= SIZE_MAX - mopts.malloc_guard - MALLOC_PAGESIZE) { 1622 if (sz >= SIZE_MAX - mopts.malloc_guard - MALLOC_PAGESIZE) {
@@ -1631,7 +1642,7 @@ omemalign(size_t alignment, size_t sz, int zero_fill, void *f)
1631 if (mopts.malloc_guard) { 1642 if (mopts.malloc_guard) {
1632 if (mprotect((char *)p + psz - mopts.malloc_guard, 1643 if (mprotect((char *)p + psz - mopts.malloc_guard,
1633 mopts.malloc_guard, PROT_NONE)) 1644 mopts.malloc_guard, PROT_NONE))
1634 wrterror("mprotect", NULL); 1645 wrterror(pool, "mprotect", NULL);
1635 STATS_ADD(pool->malloc_guarded, mopts.malloc_guard); 1646 STATS_ADD(pool->malloc_guarded, mopts.malloc_guard);
1636 } 1647 }
1637 1648
@@ -1649,6 +1660,7 @@ omemalign(size_t alignment, size_t sz, int zero_fill, void *f)
1649int 1660int
1650posix_memalign(void **memptr, size_t alignment, size_t size) 1661posix_memalign(void **memptr, size_t alignment, size_t size)
1651{ 1662{
1663 struct dir_info *d;
1652 int res, saved_errno = errno; 1664 int res, saved_errno = errno;
1653 void *r; 1665 void *r;
1654 1666
@@ -1657,23 +1669,25 @@ posix_memalign(void **memptr, size_t alignment, size_t size)
1657 return EINVAL; 1669 return EINVAL;
1658 1670
1659 _MALLOC_LOCK(); 1671 _MALLOC_LOCK();
1660 malloc_func = "posix_memalign():"; 1672 d = getpool();
1661 if (getpool() == NULL) { 1673 if (d == NULL) {
1662 if (malloc_init() != 0) 1674 if (malloc_init() != 0)
1663 goto err; 1675 goto err;
1676 d = getpool();
1664 } 1677 }
1665 if (malloc_active++) { 1678 d->func = "posix_memalign():";
1666 malloc_recurse(); 1679 if (d->active++) {
1680 malloc_recurse(d);
1667 goto err; 1681 goto err;
1668 } 1682 }
1669 if (size > 0 && size <= MALLOC_MAXCHUNK) 1683 if (size > 0 && size <= MALLOC_MAXCHUNK)
1670 size += mopts.malloc_canaries; 1684 size += mopts.malloc_canaries;
1671 r = omemalign(alignment, size, 0, CALLER); 1685 r = omemalign(d, alignment, size, 0, CALLER);
1672 malloc_active--; 1686 d->active--;
1673 _MALLOC_UNLOCK(); 1687 _MALLOC_UNLOCK();
1674 if (r == NULL) { 1688 if (r == NULL) {
1675 if (mopts.malloc_xmalloc) { 1689 if (mopts.malloc_xmalloc) {
1676 wrterror("out of memory", NULL); 1690 wrterror(d, "out of memory", NULL);
1677 errno = ENOMEM; 1691 errno = ENOMEM;
1678 } 1692 }
1679 goto err; 1693 goto err;
@@ -1925,7 +1939,7 @@ malloc_dump(int fd)
1925 continue; 1939 continue;
1926 r = find(pool, p); 1940 r = find(pool, p);
1927 if (r == NULL) { 1941 if (r == NULL) {
1928 wrterror("bogus pointer in malloc_dump", p); 1942 wrterror(pool, "bogus pointer in malloc_dump", p);
1929 continue; 1943 continue;
1930 } 1944 }
1931 free_bytes(pool, r, p); 1945 free_bytes(pool, r, p);