summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorotto <>2008-07-28 04:56:38 +0000
committerotto <>2008-07-28 04:56:38 +0000
commitb27d6e846482338e09c058638005f2b0c06b1832 (patch)
tree5414fccbabb7e78e967120c9a6cd8fe4965a07b3
parent05b581861619291083a9b67ba66338b4a17535c8 (diff)
downloadopenbsd-b27d6e846482338e09c058638005f2b0c06b1832.tar.gz
openbsd-b27d6e846482338e09c058638005f2b0c06b1832.tar.bz2
openbsd-b27d6e846482338e09c058638005f2b0c06b1832.zip
Almost complete rewrite of malloc, to have a more efficient data
structure of tracking pages returned by mmap(). Lots of testing by lots of people, thanks to you all. ok djm@ (for a slighly earlier version) deraadt@
Diffstat
-rw-r--r--src/lib/libc/stdlib/malloc.c2278
1 files changed, 835 insertions, 1443 deletions
diff --git a/src/lib/libc/stdlib/malloc.c b/src/lib/libc/stdlib/malloc.c
index 6042e47e69..ae8cd8157b 100644
--- a/src/lib/libc/stdlib/malloc.c
+++ b/src/lib/libc/stdlib/malloc.c
@@ -1,471 +1,324 @@
1/* $OpenBSD: malloc.c,v 1.91 2008/06/13 21:18:42 otto Exp $ */ 1/* $OpenBSD: malloc.c,v 1.92 2008/07/28 04:56:38 otto Exp $ */
2/*
3 * Copyright (c) 2008 Otto Moerbeek <otto@drijf.net>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
2 17
3/* 18/*
19 * Parts of this code, mainly the sub page sized chunk management code is
20 * derived from the malloc implementation with the following license:
21 */
22/*
4 * ---------------------------------------------------------------------------- 23 * ----------------------------------------------------------------------------
5 * "THE BEER-WARE LICENSE" (Revision 42): 24 * "THE BEER-WARE LICENSE" (Revision 42):
6 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 25 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you
7 * can do whatever you want with this stuff. If we meet some day, and you think 26 * can do whatever you want with this stuff. If we meet some day, and you think
8 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 27 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
9 * ---------------------------------------------------------------------------- 28 * ----------------------------------------------------------------------------
10 */ 29 */
11 30
12/* 31/* #define MALLOC_STATS */
13 * Defining MALLOC_EXTRA_SANITY will enable extra checks which are
14 * related to internal conditions and consistency in malloc.c. This has
15 * a noticeable runtime performance hit, and generally will not do you
16 * any good unless you fiddle with the internals of malloc or want
17 * to catch random pointer corruption as early as possible.
18 */
19#ifndef MALLOC_EXTRA_SANITY
20#undef MALLOC_EXTRA_SANITY
21#endif
22
23/*
24 * Defining MALLOC_STATS will enable you to call malloc_dump() and set
25 * the [dD] options in the MALLOC_OPTIONS environment variable.
26 * It has no run-time performance hit, but does pull in stdio...
27 */
28#ifndef MALLOC_STATS
29#undef MALLOC_STATS
30#endif
31
32/*
33 * What to use for Junk. This is the byte value we use to fill with
34 * when the 'J' option is enabled.
35 */
36#define SOME_JUNK 0xd0 /* as in "Duh" :-) */
37 32
38#include <sys/types.h> 33#include <sys/types.h>
39#include <sys/time.h>
40#include <sys/resource.h>
41#include <sys/param.h> 34#include <sys/param.h>
42#include <sys/mman.h> 35#include <sys/mman.h>
43#include <sys/uio.h> 36#include <sys/uio.h>
44#include <stdio.h> 37#include <errno.h>
38#include <stdint.h>
45#include <stdlib.h> 39#include <stdlib.h>
46#include <string.h> 40#include <string.h>
41#include <stdio.h>
47#include <unistd.h> 42#include <unistd.h>
43
44#ifdef MALLOC_STATS
48#include <fcntl.h> 45#include <fcntl.h>
49#include <limits.h> 46#endif
50#include <errno.h>
51#include <err.h>
52 47
53#include "thread_private.h" 48#include "thread_private.h"
54 49
55/* 50#define MALLOC_MINSHIFT 4
56 * The basic parameters you can tweak. 51#define MALLOC_MAXSHIFT 16
57 *
58 * malloc_pageshift pagesize = 1 << malloc_pageshift
59 * It's probably best if this is the native
60 * page size, but it shouldn't have to be.
61 *
62 * malloc_minsize minimum size of an allocation in bytes.
63 * If this is too small it's too much work
64 * to manage them. This is also the smallest
65 * unit of alignment used for the storage
66 * returned by malloc/realloc.
67 *
68 */
69
70#if defined(__sparc__)
71#define malloc_pageshift 13U
72#endif /* __sparc__ */
73
74#ifndef malloc_pageshift
75#define malloc_pageshift (PGSHIFT)
76#endif
77 52
78#ifndef malloc_minsize 53#if defined(__sparc__) && !defined(__sparcv9__)
79#define malloc_minsize 16UL 54#define MALLOC_PAGESHIFT (13U)
55#else
56#define MALLOC_PAGESHIFT (PGSHIFT)
80#endif 57#endif
81 58
82#if !defined(malloc_pagesize) 59#define MALLOC_PAGESIZE (1UL << MALLOC_PAGESHIFT)
83#define malloc_pagesize (1UL<<malloc_pageshift) 60#define MALLOC_MINSIZE (1UL << MALLOC_MINSHIFT)
84#endif 61#define MALLOC_PAGEMASK (MALLOC_PAGESIZE - 1)
62#define MASK_POINTER(p) ((void *)(((uintptr_t)(p)) & ~MALLOC_PAGEMASK))
85 63
86/* How many bits per u_long in the bitmap */ 64#define MALLOC_MAXCHUNK (1 << (MALLOC_PAGESHIFT-1))
87#define MALLOC_BITS (NBBY * sizeof(u_long)) 65#define MALLOC_MAXCACHE 256
66#define MALLOC_DELAYED_CHUNKS 16 /* should be power of 2 */
88 67
68#define PAGEROUND(x) (((x) + (MALLOC_PAGEMASK)) & ~MALLOC_PAGEMASK)
89 69
90/* 70/*
91 * No user serviceable parts behind this point. 71 * What to use for Junk. This is the byte value we use to fill with
92 * 72 * when the 'J' option is enabled. Use SOME_JUNK right after alloc,
93 * This structure describes a page worth of chunks. 73 * and SOME_FREEJUNK right before free.
94 */
95struct pginfo {
96 struct pginfo *next; /* next on the free list */
97 void *page; /* Pointer to the page */
98 u_short size; /* size of this page's chunks */
99 u_short shift; /* How far to shift for this size chunks */
100 u_short free; /* How many free chunks */
101 u_short total; /* How many chunk */
102 u_long bits[(malloc_pagesize / malloc_minsize) / MALLOC_BITS];/* Which chunks are free */
103};
104
105/*
106 * This structure describes a number of free pages.
107 */
108struct pgfree {
109 struct pgfree *next; /* next run of free pages */
110 struct pgfree *prev; /* prev run of free pages */
111 void *page; /* pointer to free pages */
112 void *pdir; /* pointer to the base page's dir */
113 size_t size; /* number of bytes free */
114};
115
116/*
117 * Magic values to put in the page_directory
118 */ 74 */
119#define MALLOC_NOT_MINE ((struct pginfo*) 0) 75#define SOME_JUNK 0xd0 /* as in "Duh" :-) */
120#define MALLOC_FREE ((struct pginfo*) 1) 76#define SOME_FREEJUNK 0xdf
121#define MALLOC_FIRST ((struct pginfo*) 2)
122#define MALLOC_FOLLOW ((struct pginfo*) 3)
123#define MALLOC_MAGIC ((struct pginfo*) 4)
124
125#if ((1UL<<malloc_pageshift) != malloc_pagesize)
126#error "(1UL<<malloc_pageshift) != malloc_pagesize"
127#endif
128
129#ifndef malloc_maxsize
130#define malloc_maxsize ((malloc_pagesize)>>1)
131#endif
132
133/* A mask for the offset inside a page. */
134#define malloc_pagemask ((malloc_pagesize)-1)
135
136#define pageround(foo) (((foo) + (malloc_pagemask)) & ~malloc_pagemask)
137#define ptr2index(foo) (((u_long)(foo) >> malloc_pageshift)+malloc_pageshift)
138#define index2ptr(idx) ((void*)(((idx)-malloc_pageshift)<<malloc_pageshift))
139 77
140/* Set when initialization has been done */ 78#define MMAP(sz) mmap(NULL, (size_t)(sz), PROT_READ | PROT_WRITE, \
141static unsigned int malloc_started; 79 MAP_ANON | MAP_PRIVATE, -1, (off_t) 0)
142 80
143/* Number of free pages we cache */ 81struct region_info {
144static unsigned int malloc_cache = 16; 82 void *p; /* page; low bits used to mark chunks */
145 83 uintptr_t size; /* size for pages, or chunk_info pointer */
146/* Structure used for linking discrete directory pages. */
147struct pdinfo {
148 struct pginfo **base;
149 struct pdinfo *prev;
150 struct pdinfo *next;
151 u_long dirnum;
152}; 84};
153static struct pdinfo *last_dir; /* Caches to the last and previous */
154static struct pdinfo *prev_dir; /* referenced directory pages. */
155
156static size_t pdi_off;
157static u_long pdi_mod;
158#define PD_IDX(num) ((num) / (malloc_pagesize/sizeof(struct pginfo *)))
159#define PD_OFF(num) ((num) & ((malloc_pagesize/sizeof(struct pginfo *))-1))
160#define PI_IDX(index) ((index) / pdi_mod)
161#define PI_OFF(index) ((index) % pdi_mod)
162
163/* The last index in the page directory we care about */
164static u_long last_index;
165
166/* Pointer to page directory. Allocated "as if with" malloc */
167static struct pginfo **page_dir;
168
169/* Free pages line up here */
170static struct pgfree free_list;
171
172/* Abort(), user doesn't handle problems. */
173static int malloc_abort = 2;
174
175/* Are we trying to die ? */
176static int suicide;
177 85
86struct dir_info {
87 u_int32_t canary1;
88 struct region_info *r; /* region slots */
89 size_t regions_total; /* number of region slots */
90 size_t regions_bits; /* log2 of total */
91 size_t regions_free; /* number of free slots */
92 /* list of free chunk info structs */
93 struct chunk_info *chunk_info_list;
94 /* lists of chunks with free slots */
95 struct chunk_info *chunk_dir[MALLOC_MAXSHIFT];
96 size_t free_regions_size; /* free pages cached */
97 /* free pages cache */
98 struct region_info free_regions[MALLOC_MAXCACHE];
99 /* delayed free chunk slots */
100 void *delayed_chunks[MALLOC_DELAYED_CHUNKS];
178#ifdef MALLOC_STATS 101#ifdef MALLOC_STATS
179/* dump statistics */ 102 size_t inserts;
180static int malloc_stats; 103 size_t insert_collisions;
181#endif 104 size_t finds;
182 105 size_t find_collisions;
183/* avoid outputting warnings? */ 106 size_t deletes;
184static int malloc_silent; 107 size_t delete_moves;
185 108#define STATS_INC(x) ((x)++)
186/* always realloc ? */ 109#define STATS_ZERO(x) ((x) = 0)
187static int malloc_realloc; 110#else
188 111#define STATS_INC(x) /* nothing */
189/* mprotect free pages PROT_NONE? */ 112#define STATS_ZERO(x) /* nothing */
190static int malloc_freeprot; 113#endif /* MALLOC_STATS */
191 114 u_int32_t canary2;
192/* use guard pages after allocations? */ 115};
193static size_t malloc_guard = 0;
194static size_t malloc_guarded;
195/* align pointers to end of page? */
196static int malloc_ptrguard;
197
198static int malloc_hint;
199
200/* xmalloc behaviour ? */
201static int malloc_xmalloc;
202
203/* zero fill ? */
204static int malloc_zero;
205
206/* junk fill ? */
207static int malloc_junk;
208 116
209#ifdef __FreeBSD__
210/* utrace ? */
211static int malloc_utrace;
212 117
213struct ut { 118/*
214 void *p; 119 * This structure describes a page worth of chunks.
215 size_t s; 120 *
216 void *r; 121 * How many bits per u_long in the bitmap
122 */
123#define MALLOC_BITS (NBBY * sizeof(u_long))
124struct chunk_info {
125 struct chunk_info *next; /* next on the free list */
126 void *page; /* pointer to the page */
127 u_int32_t canary;
128 u_short size; /* size of this page's chunks */
129 u_short shift; /* how far to shift for this size */
130 u_short free; /* how many free chunks */
131 u_short total; /* how many chunk */
132 /* which chunks are free */
133 u_long bits[(MALLOC_PAGESIZE / MALLOC_MINSIZE) / MALLOC_BITS];
217}; 134};
218 135
219void utrace(struct ut *, int); 136static struct dir_info g_pool;
137static char *malloc_func; /* current function */
138char *malloc_options; /* compile-time options */
139
140static int malloc_abort = 1; /* abort() on error */
141static int malloc_active; /* status of malloc */
142static int malloc_freeprot; /* mprotect free pages PROT_NONE? */
143static int malloc_hint; /* call madvice on free pages? */
144static int malloc_junk; /* junk fill? */
145static int malloc_move; /* move allocations to end of page? */
146static int malloc_realloc; /* always realloc? */
147static int malloc_silent; /* avoid outputting warnings? */
148static int malloc_xmalloc; /* xmalloc behaviour? */
149static int malloc_zero; /* zero fill? */
150static size_t malloc_guard; /* use guard pages after allocations? */
151
152static u_int malloc_cache = 64; /* free pages we cache */
153static size_t malloc_guarded; /* bytes used for guards */
154static size_t malloc_used; /* bytes allocated */
220 155
221#define UTRACE(a, b, c) \ 156#ifdef MALLOC_STATS
222 if (malloc_utrace) \ 157static int malloc_stats; /* dump statistics at end */
223 {struct ut u; u.p=a; u.s = b; u.r=c; utrace(&u, sizeof u);}
224#else /* !__FreeBSD__ */
225#define UTRACE(a,b,c)
226#endif 158#endif
227 159
228/* Status of malloc. */ 160static size_t rbytesused; /* random bytes used */
229static int malloc_active; 161static u_char rbytes[4096]; /* random bytes */
230 162static u_char getrbyte(void);
231/* Allocated memory. */
232static size_t malloc_used;
233
234/* My last break. */
235static caddr_t malloc_brk;
236 163
237/* One location cache for free-list holders. */ 164extern char *__progname;
238static struct pgfree *px;
239
240/* Compile-time options. */
241char *malloc_options;
242
243/* Name of the current public function. */
244static char *malloc_func;
245
246#define MMAP(size) \
247 mmap((void *)0, (size), PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, \
248 -1, (off_t)0)
249 165
250/* 166/* low bits of r->p determine size: 0 means >= page size and p->size holding
251 * Necessary function declarations. 167 * real size, otherwise r->size is a shift count, or 1 for malloc(0)
252 */ 168 */
253static void *imalloc(size_t size); 169#define REALSIZE(sz, r) \
254static void ifree(void *ptr); 170 (sz) = (uintptr_t)(r)->p & MALLOC_PAGEMASK, \
255static void *irealloc(void *ptr, size_t size); 171 (sz) = ((sz) == 0 ? (r)->size : ((sz) == 1 ? 0 : (1 << ((sz)-1))))
256static void *malloc_bytes(size_t size);
257
258static struct pginfo *pginfo_list;
259 172
260static struct pgfree *pgfree_list; 173static inline size_t
174hash(void *p)
175{
176 size_t sum;
177 union {
178 uintptr_t p;
179 unsigned short a[sizeof(void *) / sizeof(short)];
180 } u;
181 u.p = (uintptr_t)p >> MALLOC_PAGESHIFT;
182 sum = u.a[0];
183 sum = (sum << 7) - sum + u.a[1];
184#ifdef __LP64__
185 sum = (sum << 7) - sum + u.a[2];
186 sum = (sum << 7) - sum + u.a[3];
187#endif
188 return sum;
189}
261 190
262static struct pgfree * 191#ifdef MALLOC_STATS
263alloc_pgfree(void) 192static void
193dump_chunk(int fd, struct chunk_info *p, int fromfreelist)
264{ 194{
265 struct pgfree *p; 195 char buf[64];
266 int i;
267 196
268 if (pgfree_list == NULL) { 197 while (p) {
269 p = MMAP(malloc_pagesize); 198 snprintf(buf, sizeof(buf), "chunk %d %d/%d %p\n", p->size,
270 if (p == MAP_FAILED) 199 p->free, p->total, p->page);
271 return NULL; 200 write(fd, buf, strlen(buf));
272 for (i = 0; i < malloc_pagesize / sizeof(*p); i++) { 201 if (!fromfreelist)
273 p[i].next = pgfree_list; 202 break;
274 pgfree_list = &p[i]; 203 p = p->next;
204 if (p != NULL) {
205 snprintf(buf, sizeof(buf), " ");
206 write(fd, buf, strlen(buf));
275 } 207 }
276 } 208 }
277 p = pgfree_list;
278 pgfree_list = p->next;
279 memset(p, 0, sizeof *p);
280 return p;
281} 209}
282 210
283static struct pginfo * 211static void
284alloc_pginfo(void) 212dump_free_chunk_info(int fd, struct dir_info *d)
285{ 213{
286 struct pginfo *p; 214 char buf[64];
287 int i; 215 int i;
288 216
289 if (pginfo_list == NULL) { 217 snprintf(buf, sizeof(buf), "Free chunk structs:\n");
290 p = MMAP(malloc_pagesize); 218 write(fd, buf, strlen(buf));
291 if (p == MAP_FAILED) 219 for (i = 0; i < MALLOC_MAXSHIFT; i++) {
292 return NULL; 220 struct chunk_info *p = d->chunk_dir[i];
293 for (i = 0; i < malloc_pagesize / sizeof(*p); i++) { 221 if (p != NULL) {
294 p[i].next = pginfo_list; 222 snprintf(buf, sizeof(buf), "%2d) ", i);
295 pginfo_list = &p[i]; 223 write(fd, buf, strlen(buf));
224 dump_chunk(fd, p, 1);
296 } 225 }
297 } 226 }
298 p = pginfo_list;
299 pginfo_list = p->next;
300 memset(p, 0, sizeof *p);
301 return p;
302}
303 227
304static void
305put_pgfree(struct pgfree *p)
306{
307 p->next = pgfree_list;
308 pgfree_list = p;
309} 228}
310 229
311static void 230static void
312put_pginfo(struct pginfo *p) 231dump_free_page_info(int fd, struct dir_info *d)
313{
314 p->next = pginfo_list;
315 pginfo_list = p;
316}
317
318/*
319 * Function for page directory lookup.
320 */
321static int
322pdir_lookup(u_long index, struct pdinfo ** pdi)
323{ 232{
324 struct pdinfo *spi; 233 char buf[64];
325 u_long pidx = PI_IDX(index); 234 int i;
326
327 if (last_dir != NULL && PD_IDX(last_dir->dirnum) == pidx)
328 *pdi = last_dir;
329 else if (prev_dir != NULL && PD_IDX(prev_dir->dirnum) == pidx)
330 *pdi = prev_dir;
331 else if (last_dir != NULL && prev_dir != NULL) {
332 if ((PD_IDX(last_dir->dirnum) > pidx) ?
333 (PD_IDX(last_dir->dirnum) - pidx) :
334 (pidx - PD_IDX(last_dir->dirnum))
335 < (PD_IDX(prev_dir->dirnum) > pidx) ?
336 (PD_IDX(prev_dir->dirnum) - pidx) :
337 (pidx - PD_IDX(prev_dir->dirnum)))
338 *pdi = last_dir;
339 else
340 *pdi = prev_dir;
341
342 if (PD_IDX((*pdi)->dirnum) > pidx) {
343 for (spi = (*pdi)->prev;
344 spi != NULL && PD_IDX(spi->dirnum) > pidx;
345 spi = spi->prev)
346 *pdi = spi;
347 if (spi != NULL)
348 *pdi = spi;
349 } else
350 for (spi = (*pdi)->next;
351 spi != NULL && PD_IDX(spi->dirnum) <= pidx;
352 spi = spi->next)
353 *pdi = spi;
354 } else {
355 *pdi = (struct pdinfo *) ((caddr_t) page_dir + pdi_off);
356 for (spi = *pdi;
357 spi != NULL && PD_IDX(spi->dirnum) <= pidx;
358 spi = spi->next)
359 *pdi = spi;
360 }
361
362 return ((PD_IDX((*pdi)->dirnum) == pidx) ? 0 :
363 (PD_IDX((*pdi)->dirnum) > pidx) ? 1 : -1);
364}
365 235
366#ifdef MALLOC_STATS 236 snprintf(buf, sizeof(buf), "Free pages cached: %zu\n",
367void 237 d->free_regions_size);
368malloc_dump(int fd) 238 write(fd, buf, strlen(buf));
369{ 239 for (i = 0; i < malloc_cache; i++) {
370 char buf[1024]; 240 if (d->free_regions[i].p != NULL) {
371 struct pginfo **pd; 241 snprintf(buf, sizeof(buf), "%2d) ", i);
372 struct pgfree *pf;
373 struct pdinfo *pi;
374 u_long j;
375
376 pd = page_dir;
377 pi = (struct pdinfo *) ((caddr_t) pd + pdi_off);
378
379 /* print out all the pages */
380 for (j = 0; j <= last_index;) {
381 snprintf(buf, sizeof buf, "%08lx %5lu ", j << malloc_pageshift, j);
382 write(fd, buf, strlen(buf));
383 if (pd[PI_OFF(j)] == MALLOC_NOT_MINE) {
384 for (j++; j <= last_index && pd[PI_OFF(j)] == MALLOC_NOT_MINE;) {
385 if (!PI_OFF(++j)) {
386 if ((pi = pi->next) == NULL ||
387 PD_IDX(pi->dirnum) != PI_IDX(j))
388 break;
389 pd = pi->base;
390 j += pdi_mod;
391 }
392 }
393 j--;
394 snprintf(buf, sizeof buf, ".. %5lu not mine\n", j);
395 write(fd, buf, strlen(buf));
396 } else if (pd[PI_OFF(j)] == MALLOC_FREE) {
397 for (j++; j <= last_index && pd[PI_OFF(j)] == MALLOC_FREE;) {
398 if (!PI_OFF(++j)) {
399 if ((pi = pi->next) == NULL ||
400 PD_IDX(pi->dirnum) != PI_IDX(j))
401 break;
402 pd = pi->base;
403 j += pdi_mod;
404 }
405 }
406 j--;
407 snprintf(buf, sizeof buf, ".. %5lu free\n", j);
408 write(fd, buf, strlen(buf));
409 } else if (pd[PI_OFF(j)] == MALLOC_FIRST) {
410 for (j++; j <= last_index && pd[PI_OFF(j)] == MALLOC_FOLLOW;) {
411 if (!PI_OFF(++j)) {
412 if ((pi = pi->next) == NULL ||
413 PD_IDX(pi->dirnum) != PI_IDX(j))
414 break;
415 pd = pi->base;
416 j += pdi_mod;
417 }
418 }
419 j--;
420 snprintf(buf, sizeof buf, ".. %5lu in use\n", j);
421 write(fd, buf, strlen(buf));
422 } else if (pd[PI_OFF(j)] < MALLOC_MAGIC) {
423 snprintf(buf, sizeof buf, "(%p)\n", pd[PI_OFF(j)]);
424 write(fd, buf, strlen(buf)); 242 write(fd, buf, strlen(buf));
425 } else { 243 snprintf(buf, sizeof(buf), "free at %p: %zu\n",
426 snprintf(buf, sizeof buf, "%p %d (of %d) x %d @ %p --> %p\n", 244 d->free_regions[i].p, d->free_regions[i].size);
427 pd[PI_OFF(j)], pd[PI_OFF(j)]->free,
428 pd[PI_OFF(j)]->total, pd[PI_OFF(j)]->size,
429 pd[PI_OFF(j)]->page, pd[PI_OFF(j)]->next);
430 write(fd, buf, strlen(buf)); 245 write(fd, buf, strlen(buf));
431 } 246 }
432 if (!PI_OFF(++j)) {
433 if ((pi = pi->next) == NULL)
434 break;
435 pd = pi->base;
436 j += (1 + PD_IDX(pi->dirnum) - PI_IDX(j)) * pdi_mod;
437 }
438 } 247 }
248}
439 249
440 for (pf = free_list.next; pf; pf = pf->next) { 250static void
441 snprintf(buf, sizeof buf, "Free: @%p [%p...%p[ %ld ->%p <-%p\n", 251malloc_dump1(int fd, struct dir_info *d)
442 pf, pf->page, (char *)pf->page + pf->size, 252{
443 pf->size, pf->prev, pf->next); 253 char buf[64];
444 write(fd, buf, strlen(buf)); 254 size_t i, realsize;
445 if (pf == pf->next) {
446 snprintf(buf, sizeof buf, "Free_list loops\n");
447 write(fd, buf, strlen(buf));
448 break;
449 }
450 }
451 255
452 /* print out various info */ 256 snprintf(buf, sizeof(buf), "Malloc dir of %s at %p\n", __progname, d);
453 snprintf(buf, sizeof buf, "Minsize\t%lu\n", malloc_minsize);
454 write(fd, buf, strlen(buf)); 257 write(fd, buf, strlen(buf));
455 snprintf(buf, sizeof buf, "Maxsize\t%lu\n", malloc_maxsize); 258 snprintf(buf, sizeof(buf), "Regions slots %zu\n", d->regions_total);
456 write(fd, buf, strlen(buf)); 259 write(fd, buf, strlen(buf));
457 snprintf(buf, sizeof buf, "Pagesize\t%lu\n", malloc_pagesize); 260 snprintf(buf, sizeof(buf), "Finds %zu/%zu %f\n", d->finds,
261 d->find_collisions,
262 1.0 + (double)d->find_collisions / d->finds);
458 write(fd, buf, strlen(buf)); 263 write(fd, buf, strlen(buf));
459 snprintf(buf, sizeof buf, "Pageshift\t%u\n", malloc_pageshift); 264 snprintf(buf, sizeof(buf), "Inserts %zu/%zu %f\n", d->inserts,
265 d->insert_collisions,
266 1.0 + (double)d->insert_collisions / d->inserts);
460 write(fd, buf, strlen(buf)); 267 write(fd, buf, strlen(buf));
461 snprintf(buf, sizeof buf, "In use\t%lu\n", (u_long) malloc_used); 268 snprintf(buf, sizeof(buf), "Deletes %zu/%zu\n", d->deletes,
269 d->delete_moves);
462 write(fd, buf, strlen(buf)); 270 write(fd, buf, strlen(buf));
463 snprintf(buf, sizeof buf, "Guarded\t%lu\n", (u_long) malloc_guarded); 271 snprintf(buf, sizeof(buf), "Regions slots free %zu\n", d->regions_free);
272 write(fd, buf, strlen(buf));
273 for (i = 0; i < d->regions_total; i++) {
274 if (d->r[i].p != NULL) {
275 size_t h = hash(d->r[i].p) &
276 (d->regions_total - 1);
277 snprintf(buf, sizeof(buf), "%4zx) #%zx %zd ",
278 i, h, h - i);
279 write(fd, buf, strlen(buf));
280 REALSIZE(realsize, &d->r[i]);
281 if (realsize > MALLOC_MAXCHUNK) {
282 snprintf(buf, sizeof(buf),
283 "%p: %zu\n", d->r[i].p, realsize);
284 write(fd, buf, strlen(buf));
285 } else
286 dump_chunk(fd,
287 (struct chunk_info *)d->r[i].size, 0);
288 }
289 }
290 dump_free_chunk_info(fd, d);
291 dump_free_page_info(fd, d);
292 snprintf(buf, sizeof(buf), "In use %zu\n", malloc_used);
293 write(fd, buf, strlen(buf));
294 snprintf(buf, sizeof(buf), "Guarded %zu\n", malloc_guarded);
464 write(fd, buf, strlen(buf)); 295 write(fd, buf, strlen(buf));
465} 296}
297
298
299void
300malloc_dump(int fd)
301{
302 malloc_dump1(fd, &g_pool);
303}
304
305static void
306malloc_exit(void)
307{
308 char *q = "malloc() warning: Couldn't dump stats\n";
309 int save_errno = errno, fd;
310
311 fd = open("malloc.out", O_RDWR|O_APPEND);
312 if (fd != -1) {
313 malloc_dump(fd);
314 close(fd);
315 } else
316 write(STDERR_FILENO, q, strlen(q));
317 errno = save_errno;
318}
466#endif /* MALLOC_STATS */ 319#endif /* MALLOC_STATS */
467 320
468extern char *__progname; 321
469 322
470static void 323static void
471wrterror(char *p) 324wrterror(char *p)
@@ -485,12 +338,11 @@ wrterror(char *p)
485 iov[4].iov_len = 1; 338 iov[4].iov_len = 1;
486 writev(STDERR_FILENO, iov, 5); 339 writev(STDERR_FILENO, iov, 5);
487 340
488 suicide = 1;
489#ifdef MALLOC_STATS 341#ifdef MALLOC_STATS
490 if (malloc_stats) 342 if (malloc_stats)
491 malloc_dump(STDERR_FILENO); 343 malloc_dump(STDERR_FILENO);
492#endif /* MALLOC_STATS */ 344#endif /* MALLOC_STATS */
493 malloc_active--; 345 //malloc_active--;
494 if (malloc_abort) 346 if (malloc_abort)
495 abort(); 347 abort();
496} 348}
@@ -520,145 +372,153 @@ wrtwarning(char *p)
520 writev(STDERR_FILENO, iov, 5); 372 writev(STDERR_FILENO, iov, 5);
521} 373}
522 374
523#ifdef MALLOC_STATS
524static void
525malloc_exit(void)
526{
527 char *q = "malloc() warning: Couldn't dump stats\n";
528 int save_errno = errno, fd;
529
530 fd = open("malloc.out", O_RDWR|O_APPEND);
531 if (fd != -1) {
532 malloc_dump(fd);
533 close(fd);
534 } else
535 write(STDERR_FILENO, q, strlen(q));
536 errno = save_errno;
537}
538#endif /* MALLOC_STATS */
539
540/* 375/*
541 * Allocate a number of pages from the OS 376 * Cache maintenance. We keep at most malloc_cache pages cached.
377 * If the cache is becoming full, unmap pages in the cache for real,
378 * and then add the region to the cache
379 * Opposed to the regular region data structure, the sizes in the
380 * cache are in MALLOC_PAGESIZE units.
542 */ 381 */
543static void * 382static void
544map_pages(size_t pages) 383unmap(struct dir_info *d, void *p, size_t sz)
545{ 384{
546 struct pdinfo *pi, *spi; 385 size_t psz = PAGEROUND(sz) >> MALLOC_PAGESHIFT;
547 struct pginfo **pd; 386 size_t rsz, tounmap;
548 u_long idx, pidx, lidx; 387 struct region_info *r;
549 caddr_t result, tail; 388 u_int i, offset;
550 u_long index, lindex; 389
551 void *pdregion = NULL; 390 if (psz > malloc_cache) {
552 size_t dirs, cnt; 391 if (munmap(p, sz))
553 392 wrterror("unmap");
554 pages <<= malloc_pageshift; 393 malloc_used -= sz;
555 result = MMAP(pages + malloc_guard); 394 return;
556 if (result == MAP_FAILED) {
557#ifdef MALLOC_EXTRA_SANITY
558 wrtwarning("(ES): map_pages fails");
559#endif /* MALLOC_EXTRA_SANITY */
560 errno = ENOMEM;
561 return (NULL);
562 }
563 index = ptr2index(result);
564 tail = result + pages + malloc_guard;
565 lindex = ptr2index(tail) - 1;
566 if (malloc_guard)
567 mprotect(result + pages, malloc_guard, PROT_NONE);
568
569 pidx = PI_IDX(index);
570 lidx = PI_IDX(lindex);
571
572 if (tail > malloc_brk) {
573 malloc_brk = tail;
574 last_index = lindex;
575 }
576
577 dirs = lidx - pidx;
578
579 /* Insert directory pages, if needed. */
580 if (pdir_lookup(index, &pi) != 0)
581 dirs++;
582
583 if (dirs > 0) {
584 pdregion = MMAP(malloc_pagesize * dirs);
585 if (pdregion == MAP_FAILED) {
586 munmap(result, tail - result);
587#ifdef MALLOC_EXTRA_SANITY
588 wrtwarning("(ES): map_pages fails");
589#endif
590 errno = ENOMEM;
591 return (NULL);
592 }
593 } 395 }
594 396 tounmap = 0;
595 cnt = 0; 397 rsz = malloc_cache - d->free_regions_size;
596 for (idx = pidx, spi = pi; idx <= lidx; idx++) { 398 if (psz > rsz)
597 if (pi == NULL || PD_IDX(pi->dirnum) != idx) { 399 tounmap = psz - rsz;
598 pd = (struct pginfo **)((char *)pdregion + 400 d->free_regions_size -= tounmap;
599 cnt * malloc_pagesize); 401 offset = getrbyte();
600 cnt++; 402 for (i = 0; tounmap > 0 && i < malloc_cache; i++) {
601 memset(pd, 0, malloc_pagesize); 403 r = &d->free_regions[(i + offset) & (malloc_cache - 1)];
602 pi = (struct pdinfo *) ((caddr_t) pd + pdi_off); 404 if (r->p != NULL) {
603 pi->base = pd; 405 if (r->size <= tounmap) {
604 pi->prev = spi; 406 rsz = r->size << MALLOC_PAGESHIFT;
605 pi->next = spi->next; 407 if (munmap(r->p, rsz))
606 pi->dirnum = idx * (malloc_pagesize / 408 wrterror("munmap");
607 sizeof(struct pginfo *)); 409 tounmap -= r->size;
608 410 r->p = NULL;
609 if (spi->next != NULL) 411 r->size = 0;
610 spi->next->prev = pi; 412 malloc_used -= rsz;
611 spi->next = pi;
612 }
613 if (idx > pidx && idx < lidx) {
614 pi->dirnum += pdi_mod;
615 } else if (idx == pidx) {
616 if (pidx == lidx) {
617 pi->dirnum += (u_long)(tail - result) >>
618 malloc_pageshift;
619 } else { 413 } else {
620 pi->dirnum += pdi_mod - PI_OFF(index); 414 rsz = tounmap << MALLOC_PAGESHIFT;
415 if (munmap((char *)r->p + ((r->size - tounmap)
416 << MALLOC_PAGESHIFT), rsz))
417 wrterror("munmap");
418 r->size -= tounmap ;
419 tounmap = 0;
420 malloc_used -= rsz;
621 } 421 }
622 } else {
623 pi->dirnum += PI_OFF(ptr2index(tail - 1)) + 1;
624 } 422 }
625#ifdef MALLOC_EXTRA_SANITY 423 }
626 if (PD_OFF(pi->dirnum) > pdi_mod || PD_IDX(pi->dirnum) > idx) { 424 if (tounmap > 0)
627 wrterror("(ES): pages directory overflow"); 425 wrtwarning("malloc cache underflow");
628 errno = EFAULT; 426 for (i = 0; i < malloc_cache; i++) {
629 return (NULL); 427 r = &d->free_regions[i];
428 if (r->p == NULL) {
429 if (malloc_hint)
430 madvise(p, sz, MADV_FREE);
431 if (malloc_freeprot)
432 mprotect(p, sz, PROT_NONE);
433 r->p = p;
434 r->size = psz;
435 d->free_regions_size += psz;
436 break;
630 } 437 }
631#endif /* MALLOC_EXTRA_SANITY */ 438 }
632 if (idx == pidx && pi != last_dir) { 439 if (i == malloc_cache)
633 prev_dir = last_dir; 440 wrtwarning("malloc free slot lost");
634 last_dir = pi; 441 if (d->free_regions_size > malloc_cache)
442 wrtwarning("malloc cache overflow");
443}
444
445static void *
446map(struct dir_info *d, size_t sz)
447{
448 size_t psz = PAGEROUND(sz) >> MALLOC_PAGESHIFT;
449 struct region_info *r, *big = NULL;
450 u_int i, offset;
451 void *p;
452
453 if (psz > d->free_regions_size) {
454 p = MMAP(sz);
455 if (p != MAP_FAILED)
456 malloc_used += sz;
457 return p;
458 }
459 offset = getrbyte();
460 for (i = 0; i < malloc_cache; i++) {
461 r = &d->free_regions[(i + offset) & (malloc_cache - 1)];
462 if (r->p != NULL) {
463 if (r->size == psz) {
464 p = r->p;
465 if (malloc_freeprot)
466 mprotect(p, sz, PROT_READ | PROT_WRITE);
467 if (malloc_hint)
468 madvise(p, sz, MADV_NORMAL);
469 r->p = NULL;
470 r->size = 0;
471 d->free_regions_size -= psz;
472 return p;
473 } else if (r->size > psz)
474 big = r;
635 } 475 }
636 spi = pi;
637 pi = spi->next;
638 } 476 }
639#ifdef MALLOC_EXTRA_SANITY 477 if (big != NULL) {
640 if (cnt > dirs) 478 r = big;
641 wrtwarning("(ES): cnt > dirs"); 479 p = (char *)r->p + ((r->size - psz) << MALLOC_PAGESHIFT);
642#endif /* MALLOC_EXTRA_SANITY */ 480 if (malloc_freeprot)
643 if (cnt < dirs) 481 mprotect(p, sz, PROT_READ | PROT_WRITE);
644 munmap((char *)pdregion + cnt * malloc_pagesize, 482 if (malloc_hint)
645 (dirs - cnt) * malloc_pagesize); 483 madvise(p, sz, MADV_NORMAL);
646 484 r->size -= psz;
647 return (result); 485 d->free_regions_size -= psz;
486 return p;
487 }
488 p = MMAP(sz);
489 if (p != MAP_FAILED)
490 malloc_used += sz;
491 if (d->free_regions_size > malloc_cache)
492 wrtwarning("malloc cache");
493 return p;
494}
495
496static void
497rbytes_init(void)
498{
499 arc4random_buf(rbytes, sizeof(rbytes));
500 rbytesused = 0;
501}
502
503static u_char
504getrbyte(void)
505{
506 if (rbytesused >= sizeof(rbytes))
507 rbytes_init();
508 return rbytes[rbytesused++];
648} 509}
649 510
650/* 511/*
651 * Initialize the world 512 * Initialize a dir_info, which should have been cleared by caller
652 */ 513 */
653static void 514static int
654malloc_init(void) 515omalloc_init(struct dir_info *d)
655{ 516{
656 char *p, b[64]; 517 char *p, b[64];
657 int i, j, save_errno = errno; 518 int i, j, save_errno = errno;
519 size_t regioninfo_size;
658 520
659#ifdef MALLOC_EXTRA_SANITY 521 rbytes_init();
660 malloc_junk = 1;
661#endif /* MALLOC_EXTRA_SANITY */
662 522
663 for (i = 0; i < 3; i++) { 523 for (i = 0; i < 3; i++) {
664 switch (i) { 524 switch (i) {
@@ -686,6 +546,8 @@ malloc_init(void)
686 switch (*p) { 546 switch (*p) {
687 case '>': 547 case '>':
688 malloc_cache <<= 1; 548 malloc_cache <<= 1;
549 if (malloc_cache > MALLOC_MAXCACHE)
550 malloc_cache = MALLOC_MAXCACHE;
689 break; 551 break;
690 case '<': 552 case '<':
691 malloc_cache >>= 1; 553 malloc_cache >>= 1;
@@ -714,7 +576,7 @@ malloc_init(void)
714 malloc_guard = 0; 576 malloc_guard = 0;
715 break; 577 break;
716 case 'G': 578 case 'G':
717 malloc_guard = malloc_pagesize; 579 malloc_guard = MALLOC_PAGESIZE;
718 break; 580 break;
719 case 'h': 581 case 'h':
720 malloc_hint = 0; 582 malloc_hint = 0;
@@ -735,10 +597,10 @@ malloc_init(void)
735 malloc_silent = 1; 597 malloc_silent = 1;
736 break; 598 break;
737 case 'p': 599 case 'p':
738 malloc_ptrguard = 0; 600 malloc_move = 0;
739 break; 601 break;
740 case 'P': 602 case 'P':
741 malloc_ptrguard = 1; 603 malloc_move = 1;
742 break; 604 break;
743 case 'r': 605 case 'r':
744 malloc_realloc = 0; 606 malloc_realloc = 0;
@@ -746,14 +608,6 @@ malloc_init(void)
746 case 'R': 608 case 'R':
747 malloc_realloc = 1; 609 malloc_realloc = 1;
748 break; 610 break;
749#ifdef __FreeBSD__
750 case 'u':
751 malloc_utrace = 0;
752 break;
753 case 'U':
754 malloc_utrace = 1;
755 break;
756#endif /* __FreeBSD__ */
757 case 'x': 611 case 'x':
758 malloc_xmalloc = 0; 612 malloc_xmalloc = 0;
759 break; 613 break;
@@ -776,8 +630,6 @@ malloc_init(void)
776 } 630 }
777 } 631 }
778 632
779 UTRACE(0, 0, 0);
780
781 /* 633 /*
782 * We want junk in the entire allocation, and zero only in the part 634 * We want junk in the entire allocation, and zero only in the part
783 * the user asked for. 635 * the user asked for.
@@ -791,271 +643,228 @@ malloc_init(void)
791 " Will not be able to dump malloc stats on exit"); 643 " Will not be able to dump malloc stats on exit");
792#endif /* MALLOC_STATS */ 644#endif /* MALLOC_STATS */
793 645
794 /* Allocate one page for the page directory. */ 646 errno = save_errno;
795 page_dir = (struct pginfo **)MMAP(malloc_pagesize);
796 647
797 if (page_dir == MAP_FAILED) { 648 d->regions_bits = 9;
798 wrterror("mmap(2) failed, check limits"); 649 d->regions_free = d->regions_total = 1 << d->regions_bits;
799 errno = ENOMEM; 650 regioninfo_size = d->regions_total * sizeof(struct region_info);
800 return; 651 d->r = MMAP(regioninfo_size);
652 if (d->r == MAP_FAILED) {
653 wrterror("malloc init mmap failed");
654 d->regions_total = 0;
655 return 1;
801 } 656 }
802 pdi_off = (malloc_pagesize - sizeof(struct pdinfo)) & ~(malloc_minsize - 1); 657 malloc_used += regioninfo_size;
803 pdi_mod = pdi_off / sizeof(struct pginfo *); 658 memset(d->r, 0, regioninfo_size);
804 659 d->canary1 = arc4random();
805 last_dir = (struct pdinfo *) ((caddr_t) page_dir + pdi_off); 660 d->canary2 = ~d->canary1;
806 last_dir->base = page_dir; 661 return 0;
807 last_dir->prev = last_dir->next = NULL;
808 last_dir->dirnum = malloc_pageshift;
809
810 /* Been here, done that. */
811 malloc_started++;
812
813 /* Recalculate the cache size in bytes, and make sure it's nonzero. */
814 if (!malloc_cache)
815 malloc_cache++;
816 malloc_cache <<= malloc_pageshift;
817 errno = save_errno;
818} 662}
819 663
820/* 664static int
821 * Allocate a number of complete pages 665omalloc_grow(struct dir_info *d)
822 */
823static void *
824malloc_pages(size_t size)
825{ 666{
826 void *p, *tp; 667 size_t newbits;
827 int i; 668 size_t newtotal;
828 struct pginfo **pd; 669 size_t newsize;
829 struct pdinfo *pi; 670 size_t mask;
830 u_long pidx, index; 671 size_t i;
831 struct pgfree *pf, *delay_free = NULL; 672 struct region_info *p;
832 673
833 size = pageround(size) + malloc_guard; 674 if (d->regions_total > SIZE_MAX / sizeof(struct region_info) / 2 )
834 675 return 1;
835 p = NULL; 676
836 /* Look for free pages before asking for more */ 677 newbits = d->regions_bits + 1;
837 for (pf = free_list.next; pf; pf = pf->next) { 678 newtotal = d->regions_total * 2;
838 679 newsize = newtotal * sizeof(struct region_info);
839#ifdef MALLOC_EXTRA_SANITY 680 mask = newtotal - 1;
840 if (pf->size & malloc_pagemask) { 681
841 wrterror("(ES): junk length entry on free_list"); 682 p = MMAP(newsize);
842 errno = EFAULT; 683 if (p == MAP_FAILED)
843 return (NULL); 684 return 1;
844 } 685
845 if (!pf->size) { 686 malloc_used += newsize;
846 wrterror("(ES): zero length entry on free_list"); 687 memset(p, 0, newsize);
847 errno = EFAULT; 688 STATS_ZERO(d->inserts);
848 return (NULL); 689 STATS_ZERO(d->insert_collisions);
849 } 690 for (i = 0; i < d->regions_total; i++) {
850 if (pf->page > (pf->page + pf->size)) { 691 void *q = d->r[i].p;
851 wrterror("(ES): sick entry on free_list"); 692 if (q != NULL) {
852 errno = EFAULT; 693 size_t index = hash(q) & mask;
853 return (NULL); 694 STATS_INC(d->inserts);
854 } 695 while (p[index].p != NULL) {
855 if ((pi = pf->pdir) == NULL) { 696 index = (index - 1) & mask;
856 wrterror("(ES): invalid page directory on free-list"); 697 STATS_INC(d->insert_collisions);
857 errno = EFAULT; 698 }
858 return (NULL); 699 p[index] = d->r[i];
859 }
860 if ((pidx = PI_IDX(ptr2index(pf->page))) != PD_IDX(pi->dirnum)) {
861 wrterror("(ES): directory index mismatch on free-list");
862 errno = EFAULT;
863 return (NULL);
864 }
865 pd = pi->base;
866 if (pd[PI_OFF(ptr2index(pf->page))] != MALLOC_FREE) {
867 wrterror("(ES): non-free first page on free-list");
868 errno = EFAULT;
869 return (NULL);
870 }
871 pidx = PI_IDX(ptr2index((pf->page) + (pf->size)) - 1);
872 for (pi = pf->pdir; pi != NULL && PD_IDX(pi->dirnum) < pidx;
873 pi = pi->next)
874 ;
875 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) {
876 wrterror("(ES): last page not referenced in page directory");
877 errno = EFAULT;
878 return (NULL);
879 }
880 pd = pi->base;
881 if (pd[PI_OFF(ptr2index((pf->page) + (pf->size)) - 1)] != MALLOC_FREE) {
882 wrterror("(ES): non-free last page on free-list");
883 errno = EFAULT;
884 return (NULL);
885 }
886#endif /* MALLOC_EXTRA_SANITY */
887
888 if (pf->size < size)
889 continue;
890
891 if (pf->size == size) {
892 p = pf->page;
893 pi = pf->pdir;
894 if (pf->next != NULL)
895 pf->next->prev = pf->prev;
896 pf->prev->next = pf->next;
897 delay_free = pf;
898 break;
899 } 700 }
900 p = pf->page; 701 }
901 pf->page = (char *) pf->page + size; 702 /* avoid pages containing meta info to end up in cache */
902 pf->size -= size; 703 if (munmap(d->r, d->regions_total * sizeof(struct region_info)))
903 pidx = PI_IDX(ptr2index(pf->page)); 704 wrterror("omalloc_grow munmap");
904 for (pi = pf->pdir; pi != NULL && PD_IDX(pi->dirnum) < pidx; 705 else
905 pi = pi->next) 706 malloc_used -= d->regions_total * sizeof(struct region_info);
906 ; 707 d->regions_free = d->regions_free + d->regions_total;
907 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { 708 d->regions_total = newtotal;
908 wrterror("(ES): hole in directories"); 709 d->regions_bits = newbits;
909 errno = EFAULT; 710 d->r = p;
910 return (NULL); 711 return 0;
712}
713
714static struct chunk_info *
715alloc_chunk_info(struct dir_info *d)
716{
717 struct chunk_info *p;
718 int i;
719
720 if (d->chunk_info_list == NULL) {
721 p = MMAP(MALLOC_PAGESIZE);
722 if (p == MAP_FAILED)
723 return NULL;
724 malloc_used += MALLOC_PAGESIZE;
725 for (i = 0; i < MALLOC_PAGESIZE / sizeof(*p); i++) {
726 p[i].next = d->chunk_info_list;
727 d->chunk_info_list = &p[i];
911 } 728 }
912 tp = pf->pdir;
913 pf->pdir = pi;
914 pi = tp;
915 break;
916 } 729 }
730 p = d->chunk_info_list;
731 d->chunk_info_list = p->next;
732 memset(p, 0, sizeof *p);
733 p->canary = d->canary1;
734 return p;
735}
736
917 737
918 size -= malloc_guard; 738static void
739put_chunk_info(struct dir_info *d, struct chunk_info *p)
740{
741 p->next = d->chunk_info_list;
742 d->chunk_info_list = p;
743}
919 744
920#ifdef MALLOC_EXTRA_SANITY 745static int
921 if (p != NULL && pi != NULL) { 746insert(struct dir_info *d, void *p, size_t sz)
922 pidx = PD_IDX(pi->dirnum); 747{
923 pd = pi->base; 748 size_t index;
749 size_t mask;
750 void *q;
751
752 if (d->regions_free * 4 < d->regions_total) {
753 if (omalloc_grow(d))
754 return 1;
924 } 755 }
925 if (p != NULL && pd[PI_OFF(ptr2index(p))] != MALLOC_FREE) { 756 mask = d->regions_total - 1;
926 wrterror("(ES): allocated non-free page on free-list"); 757 index = hash(p) & mask;
927 errno = EFAULT; 758 q = d->r[index].p;
928 return (NULL); 759 STATS_INC(d->inserts);
760 while (q != NULL) {
761 index = (index - 1) & mask;
762 q = d->r[index].p;
763 STATS_INC(d->insert_collisions);
929 } 764 }
930#endif /* MALLOC_EXTRA_SANITY */ 765 d->r[index].p = p;
931 766 d->r[index].size = sz;
932 if (p != NULL && (malloc_guard || malloc_freeprot)) 767 d->regions_free--;
933 mprotect(p, size, PROT_READ | PROT_WRITE); 768 return 0;
769}
934 770
935 size >>= malloc_pageshift; 771static struct region_info *
772find(struct dir_info *d, void *p)
773{
774 size_t index;
775 size_t mask = d->regions_total - 1;
776 void *q, *r;
777
778 if (d->canary1 != ~d->canary2)
779 wrterror("internal struct corrupt");
780 p = MASK_POINTER(p);
781 index = hash(p) & mask;
782 r = d->r[index].p;
783 q = MASK_POINTER(r);
784 STATS_INC(d->finds);
785 while (q != p && r != NULL) {
786 index = (index - 1) & mask;
787 r = d->r[index].p;
788 q = MASK_POINTER(r);
789 STATS_INC(d->find_collisions);
790 }
791 return q == p ? &d->r[index] : NULL;
792}
936 793
937 /* Map new pages */ 794static void
938 if (p == NULL) 795delete(struct dir_info *d, struct region_info *ri)
939 p = map_pages(size); 796{
940 797 /* algorithm R, Knuth Vol III section 6.4 */
941 if (p != NULL) { 798 size_t mask = d->regions_total - 1;
942 index = ptr2index(p); 799 size_t i, j, r;
943 pidx = PI_IDX(index); 800
944 pdir_lookup(index, &pi); 801 if (d->regions_total & (d->regions_total - 1))
945#ifdef MALLOC_EXTRA_SANITY 802 wrterror("regions_total not 2^x");
946 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { 803 d->regions_free++;
947 wrterror("(ES): mapped pages not found in directory"); 804 STATS_INC(g_pool.deletes);
948 errno = EFAULT; 805
949 return (NULL); 806 i = ri - d->r;
950 } 807 for (;;) {
951#endif /* MALLOC_EXTRA_SANITY */ 808 d->r[i].p = NULL;
952 if (pi != last_dir) { 809 d->r[i].size = 0;
953 prev_dir = last_dir; 810 j = i;
954 last_dir = pi; 811 for (;;) {
955 } 812 i = (i - 1) & mask;
956 pd = pi->base; 813 if (d->r[i].p == NULL)
957 pd[PI_OFF(index)] = MALLOC_FIRST; 814 return;
958 for (i = 1; i < size; i++) { 815 r = hash(d->r[i].p) & mask;
959 if (!PI_OFF(index + i)) { 816 if ((i <= r && r < j) || (r < j && j < i) ||
960 pidx++; 817 (j < i && i <= r))
961 pi = pi->next; 818 continue;
962#ifdef MALLOC_EXTRA_SANITY 819 d->r[j] = d->r[i];
963 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { 820 STATS_INC(g_pool.delete_moves);
964 wrterror("(ES): hole in mapped pages directory"); 821 break;
965 errno = EFAULT;
966 return (NULL);
967 }
968#endif /* MALLOC_EXTRA_SANITY */
969 pd = pi->base;
970 }
971 pd[PI_OFF(index + i)] = MALLOC_FOLLOW;
972 }
973 if (malloc_guard) {
974 if (!PI_OFF(index + i)) {
975 pidx++;
976 pi = pi->next;
977#ifdef MALLOC_EXTRA_SANITY
978 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) {
979 wrterror("(ES): hole in mapped pages directory");
980 errno = EFAULT;
981 return (NULL);
982 }
983#endif /* MALLOC_EXTRA_SANITY */
984 pd = pi->base;
985 }
986 pd[PI_OFF(index + i)] = MALLOC_FIRST;
987 } 822 }
988 malloc_used += size << malloc_pageshift;
989 malloc_guarded += malloc_guard;
990 823
991 if (malloc_junk)
992 memset(p, SOME_JUNK, size << malloc_pageshift);
993 } 824 }
994 if (delay_free) {
995 if (px == NULL)
996 px = delay_free;
997 else
998 put_pgfree(delay_free);
999 }
1000 return (p);
1001} 825}
1002 826
1003/* 827/*
1004 * Allocate a page of fragments 828 * Allocate a page of chunks
1005 */ 829 */
1006 830static struct chunk_info *
1007static int 831omalloc_make_chunks(struct dir_info *d, int bits)
1008malloc_make_chunks(int bits)
1009{ 832{
1010 struct pginfo *bp, **pd; 833 struct chunk_info *bp;
1011 struct pdinfo *pi;
1012#ifdef MALLOC_EXTRA_SANITY
1013 u_long pidx;
1014#endif /* MALLOC_EXTRA_SANITY */
1015 void *pp; 834 void *pp;
1016 long i, k; 835 long i, k;
1017 836
1018 /* Allocate a new bucket */ 837 /* Allocate a new bucket */
1019 pp = malloc_pages((size_t)malloc_pagesize); 838 pp = map(d, MALLOC_PAGESIZE);
1020 if (pp == NULL) 839 if (pp == MAP_FAILED)
1021 return (0); 840 return NULL;
1022 841
1023 /* Find length of admin structure */ 842 bp = alloc_chunk_info(d);
1024
1025 /* Don't waste more than two chunks on this */
1026
1027 /*
1028 * If we are to allocate a memory protected page for the malloc(0)
1029 * case (when bits=0), it must be from a different page than the
1030 * pginfo page.
1031 * --> Treat it like the big chunk alloc, get a second data page.
1032 */
1033 bp = alloc_pginfo();
1034 if (bp == NULL) { 843 if (bp == NULL) {
1035 ifree(pp); 844 unmap(d, pp, MALLOC_PAGESIZE);
1036 return (0); 845 return NULL;
1037 } 846 }
1038 847
1039 /* memory protect the page allocated in the malloc(0) case */ 848 /* memory protect the page allocated in the malloc(0) case */
1040 if (bits == 0) { 849 if (bits == 0) {
1041 bp->size = 0; 850 bp->size = 0;
1042 bp->shift = 1; 851 bp->shift = 1;
1043 i = malloc_minsize - 1; 852 i = MALLOC_MINSIZE - 1;
1044 while (i >>= 1) 853 while (i >>= 1)
1045 bp->shift++; 854 bp->shift++;
1046 bp->total = bp->free = malloc_pagesize >> bp->shift; 855 bp->total = bp->free = MALLOC_PAGESIZE >> bp->shift;
1047 bp->page = pp; 856 bp->page = pp;
1048 857
1049 k = mprotect(pp, malloc_pagesize, PROT_NONE); 858 k = mprotect(pp, MALLOC_PAGESIZE, PROT_NONE);
1050 if (k < 0) { 859 if (k < 0) {
1051 ifree(pp); 860 unmap(d, pp, MALLOC_PAGESIZE);
1052 put_pginfo(bp); 861 put_chunk_info(d, bp);
1053 return (0); 862 return NULL;
1054 } 863 }
1055 } else { 864 } else {
1056 bp->size = (1UL << bits); 865 bp->size = (1UL << bits);
1057 bp->shift = bits; 866 bp->shift = bits;
1058 bp->total = bp->free = malloc_pagesize >> bits; 867 bp->total = bp->free = MALLOC_PAGESIZE >> bits;
1059 bp->page = pp; 868 bp->page = pp;
1060 } 869 }
1061 870
@@ -1070,63 +879,54 @@ malloc_make_chunks(int bits)
1070 for (; i < k; i++) 879 for (; i < k; i++)
1071 bp->bits[i / MALLOC_BITS] |= 1UL << (i % MALLOC_BITS); 880 bp->bits[i / MALLOC_BITS] |= 1UL << (i % MALLOC_BITS);
1072 881
1073 pdir_lookup(ptr2index(pp), &pi); 882 bp->next = d->chunk_dir[bits];
1074#ifdef MALLOC_EXTRA_SANITY 883 d->chunk_dir[bits] = bp;
1075 pidx = PI_IDX(ptr2index(pp));
1076 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) {
1077 wrterror("(ES): mapped pages not found in directory");
1078 errno = EFAULT;
1079 return (0);
1080 }
1081#endif /* MALLOC_EXTRA_SANITY */
1082 if (pi != last_dir) {
1083 prev_dir = last_dir;
1084 last_dir = pi;
1085 }
1086 pd = pi->base;
1087 pd[PI_OFF(ptr2index(pp))] = bp;
1088 884
1089 bp->next = page_dir[bits]; 885 bits++;
1090 page_dir[bits] = bp; 886 if ((uintptr_t)pp & bits)
887 wrterror("pp & bits");
1091 888
1092 /* MALLOC_UNLOCK */ 889 insert(d, (void *)((uintptr_t)pp | bits), (uintptr_t)bp);
1093 return (1); 890 return bp;
1094} 891}
1095 892
893
1096/* 894/*
1097 * Allocate a fragment 895 * Allocate a chunk
1098 */ 896 */
1099static void * 897static void *
1100malloc_bytes(size_t size) 898malloc_bytes(struct dir_info *d, size_t size)
1101{ 899{
1102 int i, j; 900 int i, j;
1103 size_t k; 901 size_t k;
1104 u_long u, *lp; 902 u_long u, *lp;
1105 struct pginfo *bp; 903 struct chunk_info *bp;
1106 904
1107 /* Don't bother with anything less than this */ 905 /* Don't bother with anything less than this */
1108 /* unless we have a malloc(0) requests */ 906 /* unless we have a malloc(0) requests */
1109 if (size != 0 && size < malloc_minsize) 907 if (size != 0 && size < MALLOC_MINSIZE)
1110 size = malloc_minsize; 908 size = MALLOC_MINSIZE;
1111 909
1112 /* Find the right bucket */ 910 /* Find the right bucket */
1113 if (size == 0) 911 if (size == 0)
1114 j = 0; 912 j = 0;
1115 else { 913 else {
1116 j = 1; 914 j = MALLOC_MINSHIFT;
1117 i = size - 1; 915 i = (size - 1) >> (MALLOC_MINSHIFT - 1);
1118 while (i >>= 1) 916 while (i >>= 1)
1119 j++; 917 j++;
1120 } 918 }
1121 919
1122 /* If it's empty, make a page more of that size chunks */ 920 /* If it's empty, make a page more of that size chunks */
1123 if (page_dir[j] == NULL && !malloc_make_chunks(j)) 921 bp = d->chunk_dir[j];
1124 return (NULL); 922 if (bp == NULL && (bp = omalloc_make_chunks(d, j)) == NULL)
1125 923 return NULL;
1126 bp = page_dir[j];
1127 924
925 if (bp->canary != d->canary1)
926 wrterror("chunk info corrupted");
1128 /* Find first word of bitmap which isn't empty */ 927 /* Find first word of bitmap which isn't empty */
1129 for (lp = bp->bits; !*lp; lp++); 928 for (lp = bp->bits; !*lp; lp++)
929 /* EMPTY */;
1130 930
1131 /* Find that bit, and tweak it */ 931 /* Find that bit, and tweak it */
1132 u = 1; 932 u = 1;
@@ -1136,578 +936,59 @@ malloc_bytes(size_t size)
1136 k++; 936 k++;
1137 } 937 }
1138 938
1139 if (malloc_guard) { 939 /* advance a random # of positions */
1140 /* Walk to a random position. */ 940 i = (getrbyte() & (MALLOC_DELAYED_CHUNKS - 1)) % bp->free;
1141 i = arc4random_uniform(bp->free); 941 while (i > 0) {
1142 while (i > 0) { 942 u += u;
1143 u += u; 943 k++;
1144 k++; 944 if (k >= MALLOC_BITS) {
1145 if (k >= MALLOC_BITS) { 945 lp++;
1146 lp++; 946 u = 1;
1147 u = 1; 947 k = 0;
1148 k = 0; 948 }
1149 } 949 if (lp - bp->bits > (bp->total - 1) / MALLOC_BITS) {
1150#ifdef MALLOC_EXTRA_SANITY 950 wrterror("chunk overflow");
1151 if (lp - bp->bits > (bp->total - 1) / MALLOC_BITS) { 951 errno = EFAULT;
1152 wrterror("chunk overflow"); 952 return (NULL);
1153 errno = EFAULT;
1154 return (NULL);
1155 }
1156#endif /* MALLOC_EXTRA_SANITY */
1157 if (*lp & u)
1158 i--;
1159 } 953 }
954 if (*lp & u)
955 i--;
1160 } 956 }
957
1161 *lp ^= u; 958 *lp ^= u;
1162 959
1163 /* If there are no more free, remove from free-list */ 960 /* If there are no more free, remove from free-list */
1164 if (!--bp->free) { 961 if (!--bp->free) {
1165 page_dir[j] = bp->next; 962 d->chunk_dir[j] = bp->next;
1166 bp->next = NULL; 963 bp->next = NULL;
1167 } 964 }
1168 /* Adjust to the real offset of that chunk */ 965 /* Adjust to the real offset of that chunk */
1169 k += (lp - bp->bits) * MALLOC_BITS; 966 k += (lp - bp->bits) * MALLOC_BITS;
1170 k <<= bp->shift; 967 k <<= bp->shift;
1171 968
1172 if (malloc_junk && bp->size != 0) 969 if (malloc_junk && bp->size > 0)
1173 memset((char *)bp->page + k, SOME_JUNK, (size_t)bp->size); 970 memset((char *)bp->page + k, SOME_JUNK, bp->size);
1174 971 return ((char *)bp->page + k);
1175 return ((u_char *) bp->page + k);
1176} 972}
1177 973
1178/*
1179 * Magic so that malloc(sizeof(ptr)) is near the end of the page.
1180 */
1181#define PTR_GAP (malloc_pagesize - sizeof(void *))
1182#define PTR_SIZE (sizeof(void *))
1183#define PTR_ALIGNED(p) (((unsigned long)p & malloc_pagemask) == PTR_GAP)
1184
1185/*
1186 * Allocate a piece of memory
1187 */
1188static void *
1189imalloc(size_t size)
1190{
1191 void *result;
1192 int ptralloc = 0;
1193
1194 if (!malloc_started)
1195 malloc_init();
1196
1197 if (suicide)
1198 abort();
1199
1200 /* does not matter if alloc_pgfree() fails */
1201 if (px == NULL)
1202 px = alloc_pgfree();
1203
1204 if (malloc_ptrguard && size == PTR_SIZE) {
1205 ptralloc = 1;
1206 size = malloc_pagesize;
1207 }
1208 if ((size + malloc_pagesize) < size) { /* Check for overflow */
1209 result = NULL;
1210 errno = ENOMEM;
1211 } else if (size <= malloc_maxsize)
1212 result = malloc_bytes(size);
1213 else
1214 result = malloc_pages(size);
1215
1216 if (malloc_abort == 1 && result == NULL)
1217 wrterror("allocation failed");
1218
1219 if (malloc_zero && result != NULL)
1220 memset(result, 0, size);
1221
1222 if (result && ptralloc)
1223 return ((char *) result + PTR_GAP);
1224 return (result);
1225}
1226
1227/*
1228 * Change the size of an allocation.
1229 */
1230static void *
1231irealloc(void *ptr, size_t size)
1232{
1233 void *p;
1234 size_t osize;
1235 u_long index, i;
1236 struct pginfo **mp;
1237 struct pginfo **pd;
1238 struct pdinfo *pi;
1239#ifdef MALLOC_EXTRA_SANITY
1240 u_long pidx;
1241#endif /* MALLOC_EXTRA_SANITY */
1242
1243 if (suicide)
1244 abort();
1245
1246 if (!malloc_started) {
1247 wrtwarning("malloc() has never been called");
1248 return (NULL);
1249 }
1250 if (malloc_ptrguard && PTR_ALIGNED(ptr)) {
1251 if (size <= PTR_SIZE)
1252 return (ptr);
1253
1254 p = imalloc(size);
1255 if (p)
1256 memcpy(p, ptr, PTR_SIZE);
1257 ifree(ptr);
1258 return (p);
1259 }
1260 index = ptr2index(ptr);
1261
1262 if (index < malloc_pageshift) {
1263 wrtwarning("junk pointer, too low to make sense");
1264 return (NULL);
1265 }
1266 if (index > last_index) {
1267 wrtwarning("junk pointer, too high to make sense");
1268 return (NULL);
1269 }
1270 pdir_lookup(index, &pi);
1271#ifdef MALLOC_EXTRA_SANITY
1272 pidx = PI_IDX(index);
1273 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) {
1274 wrterror("(ES): mapped pages not found in directory");
1275 errno = EFAULT;
1276 return (NULL);
1277 }
1278#endif /* MALLOC_EXTRA_SANITY */
1279 if (pi != last_dir) {
1280 prev_dir = last_dir;
1281 last_dir = pi;
1282 }
1283 pd = pi->base;
1284 mp = &pd[PI_OFF(index)];
1285
1286 if (*mp == MALLOC_FIRST) { /* Page allocation */
1287
1288 /* Check the pointer */
1289 if ((u_long) ptr & malloc_pagemask) {
1290 wrtwarning("modified (page-) pointer");
1291 return (NULL);
1292 }
1293 /* Find the size in bytes */
1294 i = index;
1295 if (!PI_OFF(++i)) {
1296 pi = pi->next;
1297 if (pi != NULL && PD_IDX(pi->dirnum) != PI_IDX(i))
1298 pi = NULL;
1299 if (pi != NULL)
1300 pd = pi->base;
1301 }
1302 for (osize = malloc_pagesize;
1303 pi != NULL && pd[PI_OFF(i)] == MALLOC_FOLLOW;) {
1304 osize += malloc_pagesize;
1305 if (!PI_OFF(++i)) {
1306 pi = pi->next;
1307 if (pi != NULL && PD_IDX(pi->dirnum) != PI_IDX(i))
1308 pi = NULL;
1309 if (pi != NULL)
1310 pd = pi->base;
1311 }
1312 }
1313
1314 if (!malloc_realloc && size <= osize &&
1315 size > osize - malloc_pagesize) {
1316 if (malloc_junk)
1317 memset((char *)ptr + size, SOME_JUNK, osize - size);
1318 return (ptr); /* ..don't do anything else. */
1319 }
1320 } else if (*mp >= MALLOC_MAGIC) { /* Chunk allocation */
1321
1322 /* Check the pointer for sane values */
1323 if ((u_long) ptr & ((1UL << ((*mp)->shift)) - 1)) {
1324 wrtwarning("modified (chunk-) pointer");
1325 return (NULL);
1326 }
1327 /* Find the chunk index in the page */
1328 i = ((u_long) ptr & malloc_pagemask) >> (*mp)->shift;
1329
1330 /* Verify that it isn't a free chunk already */
1331 if ((*mp)->bits[i / MALLOC_BITS] & (1UL << (i % MALLOC_BITS))) {
1332 wrtwarning("chunk is already free");
1333 return (NULL);
1334 }
1335 osize = (*mp)->size;
1336
1337 if (!malloc_realloc && size <= osize &&
1338 (size > osize / 2 || osize == malloc_minsize)) {
1339 if (malloc_junk)
1340 memset((char *) ptr + size, SOME_JUNK, osize - size);
1341 return (ptr); /* ..don't do anything else. */
1342 }
1343 } else {
1344 wrtwarning("irealloc: pointer to wrong page");
1345 return (NULL);
1346 }
1347
1348 p = imalloc(size);
1349
1350 if (p != NULL) {
1351 /* copy the lesser of the two sizes, and free the old one */
1352 /* Don't move from/to 0 sized region !!! */
1353 if (osize != 0 && size != 0) {
1354 if (osize < size)
1355 memcpy(p, ptr, osize);
1356 else
1357 memcpy(p, ptr, size);
1358 }
1359 ifree(ptr);
1360 }
1361 return (p);
1362}
1363
1364/*
1365 * Free a sequence of pages
1366 */
1367static void
1368free_pages(void *ptr, u_long index, struct pginfo * info)
1369{
1370 u_long i, pidx, lidx;
1371 size_t l, cachesize = 0;
1372 struct pginfo **pd;
1373 struct pdinfo *pi, *spi;
1374 struct pgfree *pf, *pt = NULL;
1375 caddr_t tail;
1376
1377 if (info == MALLOC_FREE) {
1378 wrtwarning("page is already free");
1379 return;
1380 }
1381 if (info != MALLOC_FIRST) {
1382 wrtwarning("free_pages: pointer to wrong page");
1383 return;
1384 }
1385 if ((u_long) ptr & malloc_pagemask) {
1386 wrtwarning("modified (page-) pointer");
1387 return;
1388 }
1389 /* Count how many pages and mark them free at the same time */
1390 pidx = PI_IDX(index);
1391 pdir_lookup(index, &pi);
1392#ifdef MALLOC_EXTRA_SANITY
1393 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) {
1394 wrterror("(ES): mapped pages not found in directory");
1395 errno = EFAULT;
1396 return;
1397 }
1398#endif /* MALLOC_EXTRA_SANITY */
1399
1400 spi = pi; /* Save page index for start of region. */
1401
1402 pd = pi->base;
1403 pd[PI_OFF(index)] = MALLOC_FREE;
1404 i = 1;
1405 if (!PI_OFF(index + i)) {
1406 pi = pi->next;
1407 if (pi == NULL || PD_IDX(pi->dirnum) != PI_IDX(index + i))
1408 pi = NULL;
1409 else
1410 pd = pi->base;
1411 }
1412 while (pi != NULL && pd[PI_OFF(index + i)] == MALLOC_FOLLOW) {
1413 pd[PI_OFF(index + i)] = MALLOC_FREE;
1414 i++;
1415 if (!PI_OFF(index + i)) {
1416 if ((pi = pi->next) == NULL ||
1417 PD_IDX(pi->dirnum) != PI_IDX(index + i))
1418 pi = NULL;
1419 else
1420 pd = pi->base;
1421 }
1422 }
1423
1424 l = i << malloc_pageshift;
1425
1426 if (malloc_junk)
1427 memset(ptr, SOME_JUNK, l);
1428
1429 malloc_used -= l;
1430 malloc_guarded -= malloc_guard;
1431 if (malloc_guard) {
1432#ifdef MALLOC_EXTRA_SANITY
1433 if (pi == NULL || PD_IDX(pi->dirnum) != PI_IDX(index + i)) {
1434 wrterror("(ES): hole in mapped pages directory");
1435 errno = EFAULT;
1436 return;
1437 }
1438#endif /* MALLOC_EXTRA_SANITY */
1439 pd[PI_OFF(index + i)] = MALLOC_FREE;
1440 l += malloc_guard;
1441 }
1442 tail = (caddr_t)ptr + l;
1443
1444 if (malloc_hint)
1445 madvise(ptr, l, MADV_FREE);
1446
1447 if (malloc_freeprot)
1448 mprotect(ptr, l, PROT_NONE);
1449
1450 /* Add to free-list. */
1451 if (px == NULL && (px = alloc_pgfree()) == NULL)
1452 goto not_return;
1453 px->page = ptr;
1454 px->pdir = spi;
1455 px->size = l;
1456
1457 if (free_list.next == NULL) {
1458 /* Nothing on free list, put this at head. */
1459 px->next = NULL;
1460 px->prev = &free_list;
1461 free_list.next = px;
1462 pf = px;
1463 px = NULL;
1464 } else {
1465 /*
1466 * Find the right spot, leave pf pointing to the modified
1467 * entry.
1468 */
1469
1470 /* Race ahead here, while calculating cache size. */
1471 for (pf = free_list.next;
1472 (caddr_t)ptr > ((caddr_t)pf->page + pf->size)
1473 && pf->next != NULL;
1474 pf = pf->next)
1475 cachesize += pf->size;
1476
1477 /* Finish cache size calculation. */
1478 pt = pf;
1479 while (pt) {
1480 cachesize += pt->size;
1481 pt = pt->next;
1482 }
1483
1484 if ((caddr_t)pf->page > tail) {
1485 /* Insert before entry */
1486 px->next = pf;
1487 px->prev = pf->prev;
1488 pf->prev = px;
1489 px->prev->next = px;
1490 pf = px;
1491 px = NULL;
1492 } else if (((caddr_t)pf->page + pf->size) == ptr) {
1493 /* Append to the previous entry. */
1494 cachesize -= pf->size;
1495 pf->size += l;
1496 if (pf->next != NULL &&
1497 pf->next->page == ((caddr_t)pf->page + pf->size)) {
1498 /* And collapse the next too. */
1499 pt = pf->next;
1500 pf->size += pt->size;
1501 pf->next = pt->next;
1502 if (pf->next != NULL)
1503 pf->next->prev = pf;
1504 }
1505 } else if (pf->page == tail) {
1506 /* Prepend to entry. */
1507 cachesize -= pf->size;
1508 pf->size += l;
1509 pf->page = ptr;
1510 pf->pdir = spi;
1511 } else if (pf->next == NULL) {
1512 /* Append at tail of chain. */
1513 px->next = NULL;
1514 px->prev = pf;
1515 pf->next = px;
1516 pf = px;
1517 px = NULL;
1518 } else {
1519 wrterror("freelist is destroyed");
1520 errno = EFAULT;
1521 return;
1522 }
1523 }
1524
1525 if (pf->pdir != last_dir) {
1526 prev_dir = last_dir;
1527 last_dir = pf->pdir;
1528 }
1529
1530 /* Return something to OS ? */
1531 if (pf->size > (malloc_cache - cachesize)) {
1532
1533 /*
1534 * Keep the cache intact. Notice that the '>' above guarantees that
1535 * the pf will always have at least one page afterwards.
1536 */
1537 if (munmap((char *) pf->page + (malloc_cache - cachesize),
1538 pf->size - (malloc_cache - cachesize)) != 0)
1539 goto not_return;
1540 tail = (caddr_t)pf->page + pf->size;
1541 lidx = ptr2index(tail) - 1;
1542 pf->size = malloc_cache - cachesize;
1543
1544 index = ptr2index((caddr_t)pf->page + pf->size);
1545
1546 pidx = PI_IDX(index);
1547 if (prev_dir != NULL && PD_IDX(prev_dir->dirnum) >= pidx)
1548 prev_dir = NULL; /* Will be wiped out below ! */
1549
1550 for (pi = pf->pdir; pi != NULL && PD_IDX(pi->dirnum) < pidx;
1551 pi = pi->next)
1552 ;
1553
1554 spi = pi;
1555 if (pi != NULL && PD_IDX(pi->dirnum) == pidx) {
1556 pd = pi->base;
1557
1558 for (i = index; i <= lidx;) {
1559 if (pd[PI_OFF(i)] != MALLOC_NOT_MINE) {
1560 pd[PI_OFF(i)] = MALLOC_NOT_MINE;
1561#ifdef MALLOC_EXTRA_SANITY
1562 if (!PD_OFF(pi->dirnum)) {
1563 wrterror("(ES): pages directory underflow");
1564 errno = EFAULT;
1565 return;
1566 }
1567#endif /* MALLOC_EXTRA_SANITY */
1568 pi->dirnum--;
1569 }
1570#ifdef MALLOC_EXTRA_SANITY
1571 else
1572 wrtwarning("(ES): page already unmapped");
1573#endif /* MALLOC_EXTRA_SANITY */
1574 i++;
1575 if (!PI_OFF(i)) {
1576 /*
1577 * If no page in that dir, free
1578 * directory page.
1579 */
1580 if (!PD_OFF(pi->dirnum)) {
1581 /* Remove from list. */
1582 if (spi == pi)
1583 spi = pi->prev;
1584 if (pi->prev != NULL)
1585 pi->prev->next = pi->next;
1586 if (pi->next != NULL)
1587 pi->next->prev = pi->prev;
1588 pi = pi->next;
1589 munmap(pd, malloc_pagesize);
1590 } else
1591 pi = pi->next;
1592 if (pi == NULL ||
1593 PD_IDX(pi->dirnum) != PI_IDX(i))
1594 break;
1595 pd = pi->base;
1596 }
1597 }
1598 if (pi && !PD_OFF(pi->dirnum)) {
1599 /* Resulting page dir is now empty. */
1600 /* Remove from list. */
1601 if (spi == pi) /* Update spi only if first. */
1602 spi = pi->prev;
1603 if (pi->prev != NULL)
1604 pi->prev->next = pi->next;
1605 if (pi->next != NULL)
1606 pi->next->prev = pi->prev;
1607 pi = pi->next;
1608 munmap(pd, malloc_pagesize);
1609 }
1610 }
1611 if (pi == NULL && malloc_brk == tail) {
1612 /* Resize down the malloc upper boundary. */
1613 last_index = index - 1;
1614 malloc_brk = index2ptr(index);
1615 }
1616
1617 /* XXX: We could realloc/shrink the pagedir here I guess. */
1618 if (pf->size == 0) { /* Remove from free-list as well. */
1619 if (px)
1620 put_pgfree(px);
1621 if ((px = pf->prev) != &free_list) {
1622 if (pi == NULL && last_index == (index - 1)) {
1623 if (spi == NULL) {
1624 malloc_brk = NULL;
1625 i = 11;
1626 } else {
1627 pd = spi->base;
1628 if (PD_IDX(spi->dirnum) < pidx)
1629 index =
1630 ((PD_IDX(spi->dirnum) + 1) *
1631 pdi_mod) - 1;
1632 for (pi = spi, i = index;
1633 pd[PI_OFF(i)] == MALLOC_NOT_MINE;
1634 i--) {
1635#ifdef MALLOC_EXTRA_SANITY
1636 if (!PI_OFF(i)) {
1637 pi = pi->prev;
1638 if (pi == NULL || i == 0)
1639 break;
1640 pd = pi->base;
1641 i = (PD_IDX(pi->dirnum) + 1) * pdi_mod;
1642 }
1643#endif /* MALLOC_EXTRA_SANITY */
1644 }
1645 malloc_brk = index2ptr(i + 1);
1646 }
1647 last_index = i;
1648 }
1649 if ((px->next = pf->next) != NULL)
1650 px->next->prev = px;
1651 } else {
1652 if ((free_list.next = pf->next) != NULL)
1653 free_list.next->prev = &free_list;
1654 }
1655 px = pf;
1656 last_dir = prev_dir;
1657 prev_dir = NULL;
1658 }
1659 }
1660not_return:
1661 if (pt != NULL)
1662 put_pgfree(pt);
1663}
1664 974
1665/* 975/*
1666 * Free a chunk, and possibly the page it's on, if the page becomes empty. 976 * Free a chunk, and possibly the page it's on, if the page becomes empty.
1667 */ 977 */
1668
1669/* ARGSUSED */
1670static void 978static void
1671free_bytes(void *ptr) 979free_bytes(struct dir_info *d, struct region_info *r, void *ptr)
1672{ 980{
1673 u_int8_t __arc4_getbyte(void); 981 struct chunk_info *info, **mp;
1674 struct pginfo **mp, **pd, *info; 982 long i;
1675 struct pdinfo *pi;
1676#ifdef MALLOC_EXTRA_SANITY
1677 u_long pidx;
1678#endif /* MALLOC_EXTRA_SANITY */
1679 u_long index;
1680 void *vp;
1681 long i;
1682 void *tmpptr;
1683 unsigned int tmpidx;
1684 /* pointers that we will want to free at some future time */
1685 static void *chunk_buffer[16];
1686
1687
1688 /* delay return, returning a random something from before instead */
1689 tmpidx = __arc4_getbyte() % 16;
1690 tmpptr = chunk_buffer[tmpidx];
1691 chunk_buffer[tmpidx] = ptr;
1692 ptr = tmpptr;
1693 if (!ptr)
1694 return;
1695
1696 index = ptr2index(ptr);
1697
1698 pdir_lookup(index, &pi);
1699 if (pi != last_dir) {
1700 prev_dir = last_dir;
1701 last_dir = pi;
1702 }
1703 pd = pi->base;
1704 info = pd[PI_OFF(index)];
1705 983
984 info = (struct chunk_info *)r->size;
985 if (info->canary != d->canary1)
986 wrterror("chunk info corrupted");
1706 987
1707 /* Find the chunk number on the page */ 988 /* Find the chunk number on the page */
1708 i = ((u_long) ptr & malloc_pagemask) >> info->shift; 989 i = ((uintptr_t)ptr & MALLOC_PAGEMASK) >> info->shift;
1709 990
1710 if ((u_long) ptr & ((1UL << (info->shift)) - 1)) { 991 if ((uintptr_t)ptr & ((1UL << (info->shift)) - 1)) {
1711 wrtwarning("modified (chunk-) pointer"); 992 wrtwarning("modified (chunk-) pointer");
1712 return; 993 return;
1713 } 994 }
@@ -1715,16 +996,14 @@ free_bytes(void *ptr)
1715 wrtwarning("chunk is already free"); 996 wrtwarning("chunk is already free");
1716 return; 997 return;
1717 } 998 }
1718 if (malloc_junk && info->size != 0)
1719 memset(ptr, SOME_JUNK, (size_t)info->size);
1720 999
1721 info->bits[i / MALLOC_BITS] |= 1UL << (i % MALLOC_BITS); 1000 info->bits[i / MALLOC_BITS] |= 1UL << (i % MALLOC_BITS);
1722 info->free++; 1001 info->free++;
1723 1002
1724 if (info->size != 0) 1003 if (info->size != 0)
1725 mp = page_dir + info->shift; 1004 mp = d->chunk_dir + info->shift;
1726 else 1005 else
1727 mp = page_dir; 1006 mp = d->chunk_dir;
1728 1007
1729 if (info->free == 1) { 1008 if (info->free == 1) {
1730 /* Page became non-full */ 1009 /* Page became non-full */
@@ -1743,102 +1022,71 @@ free_bytes(void *ptr)
1743 /* Find & remove this page in the queue */ 1022 /* Find & remove this page in the queue */
1744 while (*mp != info) { 1023 while (*mp != info) {
1745 mp = &((*mp)->next); 1024 mp = &((*mp)->next);
1746#ifdef MALLOC_EXTRA_SANITY
1747 if (!*mp) { 1025 if (!*mp) {
1748 wrterror("(ES): Not on queue"); 1026 wrterror("not on queue");
1749 errno = EFAULT; 1027 errno = EFAULT;
1750 return; 1028 return;
1751 } 1029 }
1752#endif /* MALLOC_EXTRA_SANITY */
1753 } 1030 }
1754 *mp = info->next; 1031 *mp = info->next;
1755 1032
1756 /* Free the page & the info structure if need be */ 1033 if (info->size == 0 && !malloc_freeprot)
1757 pdir_lookup(ptr2index(info->page), &pi); 1034 mprotect(info->page, MALLOC_PAGESIZE, PROT_READ | PROT_WRITE);
1758#ifdef MALLOC_EXTRA_SANITY 1035 unmap(d, info->page, MALLOC_PAGESIZE);
1759 pidx = PI_IDX(ptr2index(info->page));
1760 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) {
1761 wrterror("(ES): mapped pages not found in directory");
1762 errno = EFAULT;
1763 return;
1764 }
1765#endif /* MALLOC_EXTRA_SANITY */
1766 if (pi != last_dir) {
1767 prev_dir = last_dir;
1768 last_dir = pi;
1769 }
1770 pd = pi->base;
1771 pd[PI_OFF(ptr2index(info->page))] = MALLOC_FIRST;
1772
1773 /* If the page was mprotected, unprotect it before releasing it */
1774 if (info->size == 0)
1775 mprotect(info->page, malloc_pagesize, PROT_READ | PROT_WRITE);
1776 1036
1777 vp = info->page; 1037 delete(d, r);
1778 put_pginfo(info); 1038 put_chunk_info(d, info);
1779 ifree(vp);
1780} 1039}
1781 1040
1782static void
1783ifree(void *ptr)
1784{
1785 struct pginfo *info, **pd;
1786 u_long index;
1787#ifdef MALLOC_EXTRA_SANITY
1788 u_long pidx;
1789#endif /* MALLOC_EXTRA_SANITY */
1790 struct pdinfo *pi;
1791
1792 if (!malloc_started) {
1793 wrtwarning("malloc() has never been called");
1794 return;
1795 }
1796 /* If we're already sinking, don't make matters any worse. */
1797 if (suicide)
1798 return;
1799
1800 if (malloc_ptrguard && PTR_ALIGNED(ptr))
1801 ptr = (char *)ptr - PTR_GAP;
1802 1041
1803 index = ptr2index(ptr);
1804 1042
1805 if (index < malloc_pageshift) { 1043static void *
1806 warnx("(%p)", ptr); 1044omalloc(size_t sz, int zero_fill)
1807 wrtwarning("ifree: junk pointer, too low to make sense"); 1045{
1808 return; 1046 void *p;
1809 } 1047 size_t psz;
1810 if (index > last_index) {
1811 warnx("(%p)", ptr);
1812 wrtwarning("ifree: junk pointer, too high to make sense");
1813 return;
1814 }
1815 1048
1816 pdir_lookup(index, &pi); 1049 if (sz > MALLOC_MAXCHUNK) {
1817#ifdef MALLOC_EXTRA_SANITY 1050 if (sz >= SIZE_MAX - malloc_guard - MALLOC_PAGESIZE) {
1818 pidx = PI_IDX(index); 1051 errno = ENOMEM;
1819 if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { 1052 return NULL;
1820 wrterror("(ES): mapped pages not found in directory"); 1053 }
1821 errno = EFAULT; 1054 sz += malloc_guard;
1822 return; 1055 psz = PAGEROUND(sz);
1823 } 1056 p = map(&g_pool, psz);
1824#endif /* MALLOC_EXTRA_SANITY */ 1057 if (p == MAP_FAILED) {
1825 if (pi != last_dir) { 1058 errno = ENOMEM;
1826 prev_dir = last_dir; 1059 return NULL;
1827 last_dir = pi; 1060 }
1061 if (insert(&g_pool, p, sz)) {
1062 unmap(&g_pool, p, sz);
1063 errno = ENOMEM;
1064 return NULL;
1065 }
1066 if (malloc_guard) {
1067 if (mprotect((char *)p + psz - malloc_guard,
1068 malloc_guard, PROT_NONE))
1069 wrterror("mprotect");
1070 malloc_guarded += malloc_guard;
1071 }
1072 if (malloc_junk)
1073 memset(p, SOME_JUNK, psz - malloc_guard);
1074
1075 /* shift towards the end */
1076 if (malloc_move &&
1077 sz - malloc_guard < MALLOC_PAGESIZE - MALLOC_MINSIZE)
1078 p = ((char *)p) + ((MALLOC_PAGESIZE - MALLOC_MINSIZE -
1079 (sz - malloc_guard)) & ~(MALLOC_MINSIZE-1));
1080 if (zero_fill)
1081 memset(p, 0, sz - malloc_guard);
1082 } else {
1083 /* takes care of SOME_JUNK */
1084 p = malloc_bytes(&g_pool, sz);
1085 if (zero_fill && p != NULL && sz > 0)
1086 memset(p, 0, sz);
1828 } 1087 }
1829 pd = pi->base;
1830 info = pd[PI_OFF(index)];
1831
1832 if (info < MALLOC_MAGIC)
1833 free_pages(ptr, index, info);
1834 else
1835 free_bytes(ptr);
1836
1837 /* does not matter if alloc_pgfree fails */
1838 if (px == NULL)
1839 px = alloc_pgfree();
1840 1088
1841 return; 1089 return p;
1842} 1090}
1843 1091
1844/* 1092/*
@@ -1846,10 +1094,10 @@ ifree(void *ptr)
1846 * print the error message once, to avoid making the problem 1094 * print the error message once, to avoid making the problem
1847 * potentially worse. 1095 * potentially worse.
1848 */ 1096 */
1849static void 1097static void
1850malloc_recurse(void) 1098malloc_recurse(void)
1851{ 1099{
1852 static int noprint; 1100 static int noprint;
1853 1101
1854 if (noprint == 0) { 1102 if (noprint == 0) {
1855 noprint = 1; 1103 noprint = 1;
@@ -1860,74 +1108,218 @@ malloc_recurse(void)
1860 errno = EDEADLK; 1108 errno = EDEADLK;
1861} 1109}
1862 1110
1863/*
1864 * These are the public exported interface routines.
1865 */
1866void * 1111void *
1867malloc(size_t size) 1112malloc(size_t size)
1868{ 1113{
1869 void *r; 1114 void *r;
1870 1115
1871 _MALLOC_LOCK(); 1116 _MALLOC_LOCK();
1872 malloc_func = " in malloc():"; 1117 malloc_func = " in malloc():";
1118 if (!g_pool.regions_total) {
1119 if (omalloc_init(&g_pool)) {
1120 _MALLOC_UNLOCK();
1121 if (malloc_xmalloc)
1122 wrterror("out of memory");
1123 errno = ENOMEM;
1124 return NULL;
1125 }
1126 }
1873 if (malloc_active++) { 1127 if (malloc_active++) {
1874 malloc_recurse(); 1128 malloc_recurse();
1875 return (NULL); 1129 return NULL;
1876 } 1130 }
1877 r = imalloc(size); 1131 r = omalloc(size, malloc_zero);
1878 UTRACE(0, size, r);
1879 malloc_active--; 1132 malloc_active--;
1880 _MALLOC_UNLOCK(); 1133 _MALLOC_UNLOCK();
1881 if (malloc_xmalloc && r == NULL) { 1134 if (r == NULL && malloc_xmalloc) {
1882 wrterror("out of memory"); 1135 wrterror("out of memory");
1883 errno = ENOMEM; 1136 errno = ENOMEM;
1884 } 1137 }
1885 return (r); 1138 return r;
1139}
1140
1141static void
1142ofree(void *p)
1143{
1144 struct region_info *r;
1145 size_t sz;
1146
1147 r = find(&g_pool, p);
1148 if (r == NULL) {
1149 wrtwarning("bogus pointer (double free?)");
1150 return;
1151 }
1152 REALSIZE(sz, r);
1153 if (sz > MALLOC_MAXCHUNK) {
1154 if (sz - malloc_guard >= MALLOC_PAGESIZE - MALLOC_MINSIZE) {
1155 if (r->p != p)
1156 wrtwarning("bogus pointer");
1157 } else {
1158#if notyetbecause_of_realloc
1159 /* shifted towards the end */
1160 if (p != ((char *)r->p) + ((MALLOC_PAGESIZE -
1161 MALLOC_MINSIZE - sz - malloc_guard) &
1162 ~(MALLOC_MINSIZE-1))) {
1163 }
1164#endif
1165 p = r->p;
1166 }
1167 if (malloc_guard) {
1168 if (sz < malloc_guard)
1169 wrtwarning("guard size");
1170 if (!malloc_freeprot) {
1171 if (mprotect((char *)p + PAGEROUND(sz) -
1172 malloc_guard, malloc_guard,
1173 PROT_READ | PROT_WRITE))
1174 wrterror("mprotect");
1175 }
1176 malloc_guarded -= malloc_guard;
1177 }
1178 if (malloc_junk)
1179 memset(p, SOME_FREEJUNK, PAGEROUND(sz) - malloc_guard);
1180 unmap(&g_pool, p, sz);
1181 delete(&g_pool, r);
1182 } else {
1183 void *tmp;
1184 int i;
1185
1186 if (malloc_junk && sz > 0)
1187 memset(p, SOME_FREEJUNK, sz);
1188 i = getrbyte() & (MALLOC_DELAYED_CHUNKS - 1);
1189 tmp = p;
1190 p = g_pool.delayed_chunks[i];
1191 g_pool.delayed_chunks[i] = tmp;
1192 if (p != NULL) {
1193 r = find(&g_pool, p);
1194 if (r == NULL) {
1195 wrtwarning("bogus pointer (double free?)");
1196 return;
1197 }
1198 free_bytes(&g_pool, r, p);
1199 }
1200 }
1886} 1201}
1887 1202
1888void 1203void
1889free(void *ptr) 1204free(void *ptr)
1890{ 1205{
1891 /* This is legal. XXX quick path */ 1206 /* This is legal. */
1892 if (ptr == NULL) 1207 if (ptr == NULL)
1893 return; 1208 return;
1894 1209
1895 _MALLOC_LOCK(); 1210 _MALLOC_LOCK();
1896 malloc_func = " in free():"; 1211 malloc_func = " in free():";
1897 if (malloc_active++) { 1212 if (malloc_active++) {
1898 malloc_recurse(); 1213 malloc_recurse();
1899 return; 1214 return;
1900 } 1215 }
1901 ifree(ptr); 1216 ofree(ptr);
1902 UTRACE(ptr, 0, 0);
1903 malloc_active--; 1217 malloc_active--;
1904 _MALLOC_UNLOCK(); 1218 _MALLOC_UNLOCK();
1905 return; 1219}
1220
1221
1222static void *
1223orealloc(void *p, size_t newsz)
1224{
1225 struct region_info *r;
1226 size_t oldsz, goldsz, gnewsz;
1227 void *q;
1228
1229 if (p == NULL)
1230 return omalloc(newsz, 0);
1231
1232 r = find(&g_pool, p);
1233 if (r == NULL) {
1234 wrtwarning("bogus pointer (double free?)");
1235 return NULL;
1236 }
1237 if (newsz >= SIZE_MAX - malloc_guard - MALLOC_PAGESIZE) {
1238 errno = ENOMEM;
1239 return NULL;
1240 }
1241
1242 REALSIZE(oldsz, r);
1243 goldsz = oldsz;
1244 if (oldsz > MALLOC_MAXCHUNK) {
1245 if (oldsz < malloc_guard)
1246 wrtwarning("guard size");
1247 oldsz -= malloc_guard;
1248 }
1249
1250 gnewsz = newsz;
1251 if (gnewsz > MALLOC_MAXCHUNK)
1252 gnewsz += malloc_guard;
1253
1254 if (newsz > MALLOC_MAXCHUNK && oldsz > MALLOC_MAXCHUNK && p == r->p &&
1255 !malloc_realloc) {
1256 size_t roldsz = PAGEROUND(goldsz);
1257 size_t rnewsz = PAGEROUND(gnewsz);
1258
1259 if (rnewsz < roldsz) {
1260 if (malloc_guard) {
1261 if (mprotect((char *)p + roldsz - malloc_guard,
1262 malloc_guard, PROT_READ | PROT_WRITE))
1263 wrterror("mprotect");
1264 if (mprotect((char *)p + rnewsz - malloc_guard,
1265 malloc_guard, PROT_NONE))
1266 wrterror("mprotect");
1267 }
1268 unmap(&g_pool, (char *)p + rnewsz, roldsz - rnewsz);
1269 r->size = gnewsz;
1270 return p;
1271 } else if (rnewsz == roldsz) {
1272 if (newsz > oldsz && malloc_junk)
1273 memset((char *)p + newsz, SOME_JUNK,
1274 rnewsz - malloc_guard - newsz);
1275 r->size = gnewsz;
1276 return p;
1277 }
1278 }
1279 if (newsz <= oldsz && newsz > oldsz / 2 && !malloc_realloc) {
1280 if (malloc_junk && newsz > 0)
1281 memset((char *)p + newsz, SOME_JUNK, oldsz - newsz);
1282 return p;
1283 } else if (newsz != oldsz || malloc_realloc) {
1284 q = omalloc(newsz, 0);
1285 if (q == NULL)
1286 return NULL;
1287 if (newsz != 0 && oldsz != 0)
1288 memcpy(q, p, oldsz < newsz ? oldsz : newsz);
1289 ofree(p);
1290 return q;
1291 } else
1292 return p;
1906} 1293}
1907 1294
1908void * 1295void *
1909realloc(void *ptr, size_t size) 1296realloc(void *ptr, size_t size)
1910{ 1297{
1911 void *r; 1298 void *r;
1912 1299
1913 _MALLOC_LOCK(); 1300 _MALLOC_LOCK();
1914 malloc_func = " in realloc():"; 1301 malloc_func = " in realloc():";
1302 if (!g_pool.regions_total) {
1303 if (omalloc_init(&g_pool)) {
1304 _MALLOC_UNLOCK();
1305 if (malloc_xmalloc)
1306 wrterror("out of memory");
1307 errno = ENOMEM;
1308 return NULL;
1309 }
1310 }
1915 if (malloc_active++) { 1311 if (malloc_active++) {
1916 malloc_recurse(); 1312 malloc_recurse();
1917 return (NULL); 1313 return NULL;
1918 } 1314 }
1919 1315
1920 if (ptr == NULL) 1316 r = orealloc(ptr, size);
1921 r = imalloc(size); 1317
1922 else
1923 r = irealloc(ptr, size);
1924
1925 UTRACE(ptr, size, r);
1926 malloc_active--; 1318 malloc_active--;
1927 _MALLOC_UNLOCK(); 1319 _MALLOC_UNLOCK();
1928 if (malloc_xmalloc && r == NULL) { 1320 if (r == NULL && malloc_xmalloc) {
1929 wrterror("out of memory"); 1321 wrterror("out of memory");
1930 errno = ENOMEM; 1322 errno = ENOMEM;
1931 } 1323 }
1932 return (r); 1324 return r;
1933} 1325}
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-03-20 23:00:29 +0000