summaryrefslogtreecommitdiff
path: root/src/lib/libc/stdlib/malloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/libc/stdlib/malloc.c')
-rw-r--r--src/lib/libc/stdlib/malloc.c1515
1 files changed, 1515 insertions, 0 deletions
diff --git a/src/lib/libc/stdlib/malloc.c b/src/lib/libc/stdlib/malloc.c
new file mode 100644
index 0000000000..902b69c216
--- /dev/null
+++ b/src/lib/libc/stdlib/malloc.c
@@ -0,0 +1,1515 @@
1/* $OpenBSD: malloc.c,v 1.125 2010/05/18 22:24:55 tedu Exp $ */
2/*
3 * Copyright (c) 2008 Otto Moerbeek <otto@drijf.net>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18/*
19 * Parts of this code, mainly the sub page sized chunk management code is
20 * derived from the malloc implementation with the following license:
21 */
22/*
23 * ----------------------------------------------------------------------------
24 * "THE BEER-WARE LICENSE" (Revision 42):
25 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you
26 * can do whatever you want with this stuff. If we meet some day, and you think
27 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
28 * ----------------------------------------------------------------------------
29 */
30
31/* #define MALLOC_STATS */
32
33#include <sys/types.h>
34#include <sys/param.h>
35#include <sys/queue.h>
36#include <sys/mman.h>
37#include <sys/uio.h>
38#include <errno.h>
39#include <stdint.h>
40#include <stdlib.h>
41#include <string.h>
42#include <stdio.h>
43#include <unistd.h>
44
45#ifdef MALLOC_STATS
46#include <fcntl.h>
47#endif
48
49#include "thread_private.h"
50
51#define MALLOC_MINSHIFT 4
52#define MALLOC_MAXSHIFT 16
53
54#if defined(__sparc__) && !defined(__sparcv9__)
55#define MALLOC_PAGESHIFT (13U)
56#elif defined(__mips64__)
57#define MALLOC_PAGESHIFT (14U)
58#else
59#define MALLOC_PAGESHIFT (PGSHIFT)
60#endif
61
62#define MALLOC_PAGESIZE (1UL << MALLOC_PAGESHIFT)
63#define MALLOC_MINSIZE (1UL << MALLOC_MINSHIFT)
64#define MALLOC_PAGEMASK (MALLOC_PAGESIZE - 1)
65#define MASK_POINTER(p) ((void *)(((uintptr_t)(p)) & ~MALLOC_PAGEMASK))
66
67#define MALLOC_MAXCHUNK (1 << (MALLOC_PAGESHIFT-1))
68#define MALLOC_MAXCACHE 256
69#define MALLOC_DELAYED_CHUNKS 15 /* max of getrnibble() */
70/*
71 * When the P option is active, we move allocations between half a page
72 * and a whole page towards the end, subject to alignment constraints.
73 * This is the extra headroom we allow. Set to zero to be the most
74 * strict.
75 */
76#define MALLOC_LEEWAY 0
77
78#define PAGEROUND(x) (((x) + (MALLOC_PAGEMASK)) & ~MALLOC_PAGEMASK)
79
80/*
81 * What to use for Junk. This is the byte value we use to fill with
82 * when the 'J' option is enabled. Use SOME_JUNK right after alloc,
83 * and SOME_FREEJUNK right before free.
84 */
85#define SOME_JUNK 0xd0 /* as in "Duh" :-) */
86#define SOME_FREEJUNK 0xdf
87
88#define MMAP(sz) mmap(NULL, (size_t)(sz), PROT_READ | PROT_WRITE, \
89 MAP_ANON | MAP_PRIVATE, -1, (off_t) 0)
90
91#define MMAPA(a,sz) mmap((a), (size_t)(sz), PROT_READ | PROT_WRITE, \
92 MAP_ANON | MAP_PRIVATE, -1, (off_t) 0)
93
94struct region_info {
95 void *p; /* page; low bits used to mark chunks */
96 uintptr_t size; /* size for pages, or chunk_info pointer */
97};
98
99LIST_HEAD(chunk_head, chunk_info);
100
101struct dir_info {
102 u_int32_t canary1;
103 struct region_info *r; /* region slots */
104 size_t regions_total; /* number of region slots */
105 size_t regions_bits; /* log2 of total */
106 size_t regions_free; /* number of free slots */
107 /* list of free chunk info structs */
108 struct chunk_head chunk_info_list;
109 /* lists of chunks with free slots */
110 struct chunk_head chunk_dir[MALLOC_MAXSHIFT];
111 size_t free_regions_size; /* free pages cached */
112 /* free pages cache */
113 struct region_info free_regions[MALLOC_MAXCACHE];
114 /* delayed free chunk slots */
115 void *delayed_chunks[MALLOC_DELAYED_CHUNKS + 1];
116#ifdef MALLOC_STATS
117 size_t inserts;
118 size_t insert_collisions;
119 size_t finds;
120 size_t find_collisions;
121 size_t deletes;
122 size_t delete_moves;
123 size_t cheap_realloc_tries;
124 size_t cheap_reallocs;
125#define STATS_INC(x) ((x)++)
126#define STATS_ZERO(x) ((x) = 0)
127#else
128#define STATS_INC(x) /* nothing */
129#define STATS_ZERO(x) /* nothing */
130#endif /* MALLOC_STATS */
131 u_int32_t canary2;
132};
133#define DIR_INFO_RSZ ((sizeof(struct dir_info) + MALLOC_PAGEMASK) & \
134 ~MALLOC_PAGEMASK)
135
136/*
137 * This structure describes a page worth of chunks.
138 *
139 * How many bits per u_long in the bitmap
140 */
141#define MALLOC_BITS (NBBY * sizeof(u_long))
142struct chunk_info {
143 LIST_ENTRY(chunk_info) entries;
144 void *page; /* pointer to the page */
145 u_int32_t canary;
146 u_short size; /* size of this page's chunks */
147 u_short shift; /* how far to shift for this size */
148 u_short free; /* how many free chunks */
149 u_short total; /* how many chunk */
150 /* which chunks are free */
151 u_long bits[(MALLOC_PAGESIZE / MALLOC_MINSIZE) / MALLOC_BITS];
152};
153
154struct malloc_readonly {
155 struct dir_info *g_pool; /* Main bookkeeping information */
156 int malloc_abort; /* abort() on error */
157 int malloc_freeprot; /* mprotect free pages PROT_NONE? */
158 int malloc_hint; /* call madvice on free pages? */
159 int malloc_junk; /* junk fill? */
160 int malloc_move; /* move allocations to end of page? */
161 int malloc_realloc; /* always realloc? */
162 int malloc_xmalloc; /* xmalloc behaviour? */
163 int malloc_zero; /* zero fill? */
164 size_t malloc_guard; /* use guard pages after allocations? */
165 u_int malloc_cache; /* free pages we cache */
166#ifdef MALLOC_STATS
167 int malloc_stats; /* dump statistics at end */
168#endif
169 u_int32_t malloc_canary; /* Matched against ones in g_pool */
170};
171
172/* This object is mapped PROT_READ after initialisation to prevent tampering */
173static union {
174 struct malloc_readonly mopts;
175 u_char _pad[MALLOC_PAGESIZE];
176} malloc_readonly __attribute__((aligned(MALLOC_PAGESIZE)));
177#define mopts malloc_readonly.mopts
178#define g_pool mopts.g_pool
179
180char *malloc_options; /* compile-time options */
181
182static char *malloc_func; /* current function */
183static int malloc_active; /* status of malloc */
184
185static size_t malloc_guarded; /* bytes used for guards */
186static size_t malloc_used; /* bytes allocated */
187
188static size_t rnibblesused; /* random nibbles used */
189static u_char rbytes[512]; /* random bytes */
190static u_char getrnibble(void);
191
192extern char *__progname;
193
194/* low bits of r->p determine size: 0 means >= page size and p->size holding
195 * real size, otherwise r->size is a shift count, or 1 for malloc(0)
196 */
197#define REALSIZE(sz, r) \
198 (sz) = (uintptr_t)(r)->p & MALLOC_PAGEMASK, \
199 (sz) = ((sz) == 0 ? (r)->size : ((sz) == 1 ? 0 : (1 << ((sz)-1))))
200
201static inline size_t
202hash(void *p)
203{
204 size_t sum;
205 union {
206 uintptr_t p;
207 unsigned short a[sizeof(void *) / sizeof(short)];
208 } u;
209 u.p = (uintptr_t)p >> MALLOC_PAGESHIFT;
210 sum = u.a[0];
211 sum = (sum << 7) - sum + u.a[1];
212#ifdef __LP64__
213 sum = (sum << 7) - sum + u.a[2];
214 sum = (sum << 7) - sum + u.a[3];
215#endif
216 return sum;
217}
218
219#ifdef MALLOC_STATS
220static void
221dump_chunk(int fd, struct chunk_info *p, int fromfreelist)
222{
223 char buf[64];
224
225 while (p != NULL) {
226 snprintf(buf, sizeof(buf), "chunk %d %d/%d %p\n", p->size,
227 p->free, p->total, p->page);
228 write(fd, buf, strlen(buf));
229 if (!fromfreelist)
230 break;
231 p = LIST_NEXT(p, entries);
232 if (p != NULL) {
233 snprintf(buf, sizeof(buf), " ");
234 write(fd, buf, strlen(buf));
235 }
236 }
237}
238
239static void
240dump_free_chunk_info(int fd, struct dir_info *d)
241{
242 char buf[64];
243 int i;
244
245 snprintf(buf, sizeof(buf), "Free chunk structs:\n");
246 write(fd, buf, strlen(buf));
247 for (i = 0; i < MALLOC_MAXSHIFT; i++) {
248 struct chunk_info *p = LIST_FIRST(&d->chunk_dir[i]);
249 if (p != NULL) {
250 snprintf(buf, sizeof(buf), "%2d) ", i);
251 write(fd, buf, strlen(buf));
252 dump_chunk(fd, p, 1);
253 }
254 }
255
256}
257
258static void
259dump_free_page_info(int fd, struct dir_info *d)
260{
261 char buf[64];
262 int i;
263
264 snprintf(buf, sizeof(buf), "Free pages cached: %zu\n",
265 d->free_regions_size);
266 write(fd, buf, strlen(buf));
267 for (i = 0; i < mopts.malloc_cache; i++) {
268 if (d->free_regions[i].p != NULL) {
269 snprintf(buf, sizeof(buf), "%2d) ", i);
270 write(fd, buf, strlen(buf));
271 snprintf(buf, sizeof(buf), "free at %p: %zu\n",
272 d->free_regions[i].p, d->free_regions[i].size);
273 write(fd, buf, strlen(buf));
274 }
275 }
276}
277
278static void
279malloc_dump1(int fd, struct dir_info *d)
280{
281 char buf[64];
282 size_t i, realsize;
283
284 snprintf(buf, sizeof(buf), "Malloc dir of %s at %p\n", __progname, d);
285 write(fd, buf, strlen(buf));
286 if (d == NULL)
287 return;
288 snprintf(buf, sizeof(buf), "Regions slots %zu\n", d->regions_total);
289 write(fd, buf, strlen(buf));
290 snprintf(buf, sizeof(buf), "Finds %zu/%zu %f\n", d->finds,
291 d->find_collisions,
292 1.0 + (double)d->find_collisions / d->finds);
293 write(fd, buf, strlen(buf));
294 snprintf(buf, sizeof(buf), "Inserts %zu/%zu %f\n", d->inserts,
295 d->insert_collisions,
296 1.0 + (double)d->insert_collisions / d->inserts);
297 write(fd, buf, strlen(buf));
298 snprintf(buf, sizeof(buf), "Deletes %zu/%zu\n", d->deletes,
299 d->delete_moves);
300 write(fd, buf, strlen(buf));
301 snprintf(buf, sizeof(buf), "Cheap reallocs %zu/%zu\n",
302 d->cheap_reallocs, d->cheap_realloc_tries);
303 write(fd, buf, strlen(buf));
304 snprintf(buf, sizeof(buf), "Regions slots free %zu\n", d->regions_free);
305 write(fd, buf, strlen(buf));
306 for (i = 0; i < d->regions_total; i++) {
307 if (d->r[i].p != NULL) {
308 size_t h = hash(d->r[i].p) &
309 (d->regions_total - 1);
310 snprintf(buf, sizeof(buf), "%4zx) #%zx %zd ",
311 i, h, h - i);
312 write(fd, buf, strlen(buf));
313 REALSIZE(realsize, &d->r[i]);
314 if (realsize > MALLOC_MAXCHUNK) {
315 snprintf(buf, sizeof(buf),
316 "%p: %zu\n", d->r[i].p, realsize);
317 write(fd, buf, strlen(buf));
318 } else
319 dump_chunk(fd,
320 (struct chunk_info *)d->r[i].size, 0);
321 }
322 }
323 dump_free_chunk_info(fd, d);
324 dump_free_page_info(fd, d);
325 snprintf(buf, sizeof(buf), "In use %zu\n", malloc_used);
326 write(fd, buf, strlen(buf));
327 snprintf(buf, sizeof(buf), "Guarded %zu\n", malloc_guarded);
328 write(fd, buf, strlen(buf));
329}
330
331
332void
333malloc_dump(int fd)
334{
335 malloc_dump1(fd, g_pool);
336}
337
338static void
339malloc_exit(void)
340{
341 static const char q[] = "malloc() warning: Couldn't dump stats\n";
342 int save_errno = errno, fd;
343
344 fd = open("malloc.out", O_RDWR|O_APPEND);
345 if (fd != -1) {
346 malloc_dump(fd);
347 close(fd);
348 } else
349 write(STDERR_FILENO, q, sizeof(q) - 1);
350 errno = save_errno;
351}
352#endif /* MALLOC_STATS */
353
354
355
356static void
357wrterror(char *p)
358{
359 char *q = " error: ";
360 struct iovec iov[5];
361
362 iov[0].iov_base = __progname;
363 iov[0].iov_len = strlen(__progname);
364 iov[1].iov_base = malloc_func;
365 iov[1].iov_len = strlen(malloc_func);
366 iov[2].iov_base = q;
367 iov[2].iov_len = strlen(q);
368 iov[3].iov_base = p;
369 iov[3].iov_len = strlen(p);
370 iov[4].iov_base = "\n";
371 iov[4].iov_len = 1;
372 writev(STDERR_FILENO, iov, 5);
373
374#ifdef MALLOC_STATS
375 if (mopts.malloc_stats)
376 malloc_dump(STDERR_FILENO);
377#endif /* MALLOC_STATS */
378 //malloc_active--;
379 if (mopts.malloc_abort)
380 abort();
381}
382
383static void
384rbytes_init(void)
385{
386 arc4random_buf(rbytes, sizeof(rbytes));
387 rnibblesused = 0;
388}
389
390static inline u_char
391getrnibble(void)
392{
393 u_char x;
394
395 if (rnibblesused >= 2 * sizeof(rbytes))
396 rbytes_init();
397 x = rbytes[rnibblesused++ / 2];
398 return (rnibblesused & 1 ? x & 0xf : x >> 4);
399}
400
401/*
402 * Cache maintenance. We keep at most malloc_cache pages cached.
403 * If the cache is becoming full, unmap pages in the cache for real,
404 * and then add the region to the cache
405 * Opposed to the regular region data structure, the sizes in the
406 * cache are in MALLOC_PAGESIZE units.
407 */
408static void
409unmap(struct dir_info *d, void *p, size_t sz)
410{
411 size_t psz = sz >> MALLOC_PAGESHIFT;
412 size_t rsz, tounmap;
413 struct region_info *r;
414 u_int i, offset;
415
416 if (sz != PAGEROUND(sz)) {
417 wrterror("munmap round");
418 return;
419 }
420
421 if (psz > mopts.malloc_cache) {
422 if (munmap(p, sz))
423 wrterror("munmap");
424 malloc_used -= sz;
425 return;
426 }
427 tounmap = 0;
428 rsz = mopts.malloc_cache - d->free_regions_size;
429 if (psz > rsz)
430 tounmap = psz - rsz;
431 offset = getrnibble();
432 for (i = 0; tounmap > 0 && i < mopts.malloc_cache; i++) {
433 r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)];
434 if (r->p != NULL) {
435 rsz = r->size << MALLOC_PAGESHIFT;
436 if (munmap(r->p, rsz))
437 wrterror("munmap");
438 r->p = NULL;
439 if (tounmap > r->size)
440 tounmap -= r->size;
441 else
442 tounmap = 0;
443 d->free_regions_size -= r->size;
444 r->size = 0;
445 malloc_used -= rsz;
446 }
447 }
448 if (tounmap > 0)
449 wrterror("malloc cache underflow");
450 for (i = 0; i < mopts.malloc_cache; i++) {
451 r = &d->free_regions[i];
452 if (r->p == NULL) {
453 if (mopts.malloc_hint)
454 madvise(p, sz, MADV_FREE);
455 if (mopts.malloc_freeprot)
456 mprotect(p, sz, PROT_NONE);
457 r->p = p;
458 r->size = psz;
459 d->free_regions_size += psz;
460 break;
461 }
462 }
463 if (i == mopts.malloc_cache)
464 wrterror("malloc free slot lost");
465 if (d->free_regions_size > mopts.malloc_cache)
466 wrterror("malloc cache overflow");
467}
468
469static void
470zapcacheregion(struct dir_info *d, void *p)
471{
472 u_int i;
473 struct region_info *r;
474 size_t rsz;
475
476 for (i = 0; i < mopts.malloc_cache; i++) {
477 r = &d->free_regions[i];
478 if (r->p == p) {
479 rsz = r->size << MALLOC_PAGESHIFT;
480 if (munmap(r->p, rsz))
481 wrterror("munmap");
482 r->p = NULL;
483 d->free_regions_size -= r->size;
484 r->size = 0;
485 malloc_used -= rsz;
486 }
487 }
488}
489
490static void *
491map(struct dir_info *d, size_t sz, int zero_fill)
492{
493 size_t psz = sz >> MALLOC_PAGESHIFT;
494 struct region_info *r, *big = NULL;
495 u_int i, offset;
496 void *p;
497
498 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
499 d->canary1 != ~d->canary2)
500 wrterror("internal struct corrupt");
501 if (sz != PAGEROUND(sz)) {
502 wrterror("map round");
503 return NULL;
504 }
505 if (psz > d->free_regions_size) {
506 p = MMAP(sz);
507 if (p != MAP_FAILED)
508 malloc_used += sz;
509 /* zero fill not needed */
510 return p;
511 }
512 offset = getrnibble();
513 for (i = 0; i < mopts.malloc_cache; i++) {
514 r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)];
515 if (r->p != NULL) {
516 if (r->size == psz) {
517 p = r->p;
518 if (mopts.malloc_freeprot)
519 mprotect(p, sz, PROT_READ | PROT_WRITE);
520 if (mopts.malloc_hint)
521 madvise(p, sz, MADV_NORMAL);
522 r->p = NULL;
523 r->size = 0;
524 d->free_regions_size -= psz;
525 if (zero_fill)
526 memset(p, 0, sz);
527 else if (mopts.malloc_junk &&
528 mopts.malloc_freeprot)
529 memset(p, SOME_FREEJUNK, sz);
530 return p;
531 } else if (r->size > psz)
532 big = r;
533 }
534 }
535 if (big != NULL) {
536 r = big;
537 p = (char *)r->p + ((r->size - psz) << MALLOC_PAGESHIFT);
538 if (mopts.malloc_freeprot)
539 mprotect(p, sz, PROT_READ | PROT_WRITE);
540 if (mopts.malloc_hint)
541 madvise(p, sz, MADV_NORMAL);
542 r->size -= psz;
543 d->free_regions_size -= psz;
544 if (zero_fill)
545 memset(p, 0, sz);
546 else if (mopts.malloc_junk && mopts.malloc_freeprot)
547 memset(p, SOME_FREEJUNK, sz);
548 return p;
549 }
550 p = MMAP(sz);
551 if (p != MAP_FAILED)
552 malloc_used += sz;
553 if (d->free_regions_size > mopts.malloc_cache)
554 wrterror("malloc cache");
555 /* zero fill not needed */
556 return p;
557}
558
559/*
560 * Initialize a dir_info, which should have been cleared by caller
561 */
562static int
563omalloc_init(struct dir_info **dp)
564{
565 char *p, b[64];
566 int i, j;
567 size_t d_avail, regioninfo_size;
568 struct dir_info *d;
569
570 rbytes_init();
571
572 /*
573 * Default options
574 */
575 mopts.malloc_abort = 1;
576 mopts.malloc_move = 1;
577 mopts.malloc_cache = 64;
578
579 for (i = 0; i < 3; i++) {
580 switch (i) {
581 case 0:
582 j = readlink("/etc/malloc.conf", b, sizeof b - 1);
583 if (j <= 0)
584 continue;
585 b[j] = '\0';
586 p = b;
587 break;
588 case 1:
589 if (issetugid() == 0)
590 p = getenv("MALLOC_OPTIONS");
591 else
592 continue;
593 break;
594 case 2:
595 p = malloc_options;
596 break;
597 default:
598 p = NULL;
599 }
600
601 for (; p != NULL && *p != '\0'; p++) {
602 switch (*p) {
603 case '>':
604 mopts.malloc_cache <<= 1;
605 if (mopts.malloc_cache > MALLOC_MAXCACHE)
606 mopts.malloc_cache = MALLOC_MAXCACHE;
607 break;
608 case '<':
609 mopts.malloc_cache >>= 1;
610 break;
611 case 'a':
612 mopts.malloc_abort = 0;
613 break;
614 case 'A':
615 mopts.malloc_abort = 1;
616 break;
617#ifdef MALLOC_STATS
618 case 'd':
619 mopts.malloc_stats = 0;
620 break;
621 case 'D':
622 mopts.malloc_stats = 1;
623 break;
624#endif /* MALLOC_STATS */
625 case 'f':
626 mopts.malloc_freeprot = 0;
627 break;
628 case 'F':
629 mopts.malloc_freeprot = 1;
630 break;
631 case 'g':
632 mopts.malloc_guard = 0;
633 break;
634 case 'G':
635 mopts.malloc_guard = MALLOC_PAGESIZE;
636 break;
637 case 'h':
638 mopts.malloc_hint = 0;
639 break;
640 case 'H':
641 mopts.malloc_hint = 1;
642 break;
643 case 'j':
644 mopts.malloc_junk = 0;
645 break;
646 case 'J':
647 mopts.malloc_junk = 1;
648 break;
649 case 'n':
650 case 'N':
651 break;
652 case 'p':
653 mopts.malloc_move = 0;
654 break;
655 case 'P':
656 mopts.malloc_move = 1;
657 break;
658 case 'r':
659 mopts.malloc_realloc = 0;
660 break;
661 case 'R':
662 mopts.malloc_realloc = 1;
663 break;
664 case 'S':
665 mopts.malloc_freeprot = mopts.malloc_junk = 1;
666 mopts.malloc_guard = MALLOC_PAGESIZE;
667 break;
668 case 'x':
669 mopts.malloc_xmalloc = 0;
670 break;
671 case 'X':
672 mopts.malloc_xmalloc = 1;
673 break;
674 case 'z':
675 mopts.malloc_zero = 0;
676 break;
677 case 'Z':
678 mopts.malloc_zero = 1;
679 break;
680 default: {
681 static const char q[] = "malloc() warning: "
682 "unknown char in MALLOC_OPTIONS\n";
683 write(STDERR_FILENO, q, sizeof(q) - 1);
684 break;
685 }
686 }
687 }
688 }
689
690 /*
691 * We want junk in the entire allocation, and zero only in the part
692 * the user asked for.
693 */
694 if (mopts.malloc_zero)
695 mopts.malloc_junk = 1;
696
697#ifdef MALLOC_STATS
698 if (mopts.malloc_stats && (atexit(malloc_exit) == -1)) {
699 static const char q[] = "malloc() warning: atexit(2) failed."
700 " Will not be able to dump stats on exit\n";
701 write(STDERR_FILENO, q, sizeof(q) - 1);
702 }
703#endif /* MALLOC_STATS */
704
705 while ((mopts.malloc_canary = arc4random()) == 0)
706 ;
707
708 /*
709 * Allocate dir_info with a guard page on either side. Also
710 * randomise offset inside the page at which the dir_info
711 * lies (subject to alignment by 1 << MALLOC_MINSHIFT)
712 */
713 if ((p = MMAP(DIR_INFO_RSZ + (MALLOC_PAGESIZE * 2))) == MAP_FAILED)
714 return -1;
715 mprotect(p, MALLOC_PAGESIZE, PROT_NONE);
716 mprotect(p + MALLOC_PAGESIZE + DIR_INFO_RSZ,
717 MALLOC_PAGESIZE, PROT_NONE);
718 d_avail = (DIR_INFO_RSZ - sizeof(*d)) >> MALLOC_MINSHIFT;
719 d = (struct dir_info *)(p + MALLOC_PAGESIZE +
720 (arc4random_uniform(d_avail) << MALLOC_MINSHIFT));
721
722 d->regions_bits = 9;
723 d->regions_free = d->regions_total = 1 << d->regions_bits;
724 regioninfo_size = d->regions_total * sizeof(struct region_info);
725 d->r = MMAP(regioninfo_size);
726 if (d->r == MAP_FAILED) {
727 wrterror("malloc init mmap failed");
728 d->regions_total = 0;
729 return 1;
730 }
731 LIST_INIT(&d->chunk_info_list);
732 for (i = 0; i < MALLOC_MAXSHIFT; i++)
733 LIST_INIT(&d->chunk_dir[i]);
734 malloc_used += regioninfo_size;
735 d->canary1 = mopts.malloc_canary ^ (u_int32_t)(uintptr_t)d;
736 d->canary2 = ~d->canary1;
737
738 *dp = d;
739
740 /*
741 * Options have been set and will never be reset.
742 * Prevent further tampering with them.
743 */
744 if (((uintptr_t)&malloc_readonly & MALLOC_PAGEMASK) == 0)
745 mprotect(&malloc_readonly, sizeof(malloc_readonly), PROT_READ);
746
747 return 0;
748}
749
750static int
751omalloc_grow(struct dir_info *d)
752{
753 size_t newbits;
754 size_t newtotal;
755 size_t newsize;
756 size_t mask;
757 size_t i;
758 struct region_info *p;
759
760 if (d->regions_total > SIZE_MAX / sizeof(struct region_info) / 2 )
761 return 1;
762
763 newbits = d->regions_bits + 1;
764 newtotal = d->regions_total * 2;
765 newsize = newtotal * sizeof(struct region_info);
766 mask = newtotal - 1;
767
768 p = MMAP(newsize);
769 if (p == MAP_FAILED)
770 return 1;
771
772 malloc_used += newsize;
773 memset(p, 0, newsize);
774 STATS_ZERO(d->inserts);
775 STATS_ZERO(d->insert_collisions);
776 for (i = 0; i < d->regions_total; i++) {
777 void *q = d->r[i].p;
778 if (q != NULL) {
779 size_t index = hash(q) & mask;
780 STATS_INC(d->inserts);
781 while (p[index].p != NULL) {
782 index = (index - 1) & mask;
783 STATS_INC(d->insert_collisions);
784 }
785 p[index] = d->r[i];
786 }
787 }
788 /* avoid pages containing meta info to end up in cache */
789 if (munmap(d->r, d->regions_total * sizeof(struct region_info)))
790 wrterror("munmap");
791 else
792 malloc_used -= d->regions_total * sizeof(struct region_info);
793 d->regions_free = d->regions_free + d->regions_total;
794 d->regions_total = newtotal;
795 d->regions_bits = newbits;
796 d->r = p;
797 return 0;
798}
799
800static struct chunk_info *
801alloc_chunk_info(struct dir_info *d)
802{
803 struct chunk_info *p;
804 int i;
805
806 if (LIST_EMPTY(&d->chunk_info_list)) {
807 p = MMAP(MALLOC_PAGESIZE);
808 if (p == MAP_FAILED)
809 return NULL;
810 malloc_used += MALLOC_PAGESIZE;
811 for (i = 0; i < MALLOC_PAGESIZE / sizeof(*p); i++)
812 LIST_INSERT_HEAD(&d->chunk_info_list, &p[i], entries);
813 }
814 p = LIST_FIRST(&d->chunk_info_list);
815 LIST_REMOVE(p, entries);
816 memset(p, 0, sizeof *p);
817 p->canary = d->canary1;
818 return p;
819}
820
821static int
822insert(struct dir_info *d, void *p, size_t sz)
823{
824 size_t index;
825 size_t mask;
826 void *q;
827
828 if (d->regions_free * 4 < d->regions_total) {
829 if (omalloc_grow(d))
830 return 1;
831 }
832 mask = d->regions_total - 1;
833 index = hash(p) & mask;
834 q = d->r[index].p;
835 STATS_INC(d->inserts);
836 while (q != NULL) {
837 index = (index - 1) & mask;
838 q = d->r[index].p;
839 STATS_INC(d->insert_collisions);
840 }
841 d->r[index].p = p;
842 d->r[index].size = sz;
843 d->regions_free--;
844 return 0;
845}
846
847static struct region_info *
848find(struct dir_info *d, void *p)
849{
850 size_t index;
851 size_t mask = d->regions_total - 1;
852 void *q, *r;
853
854 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
855 d->canary1 != ~d->canary2)
856 wrterror("internal struct corrupt");
857 p = MASK_POINTER(p);
858 index = hash(p) & mask;
859 r = d->r[index].p;
860 q = MASK_POINTER(r);
861 STATS_INC(d->finds);
862 while (q != p && r != NULL) {
863 index = (index - 1) & mask;
864 r = d->r[index].p;
865 q = MASK_POINTER(r);
866 STATS_INC(d->find_collisions);
867 }
868 return q == p ? &d->r[index] : NULL;
869}
870
871static void
872delete(struct dir_info *d, struct region_info *ri)
873{
874 /* algorithm R, Knuth Vol III section 6.4 */
875 size_t mask = d->regions_total - 1;
876 size_t i, j, r;
877
878 if (d->regions_total & (d->regions_total - 1))
879 wrterror("regions_total not 2^x");
880 d->regions_free++;
881 STATS_INC(g_pool->deletes);
882
883 i = ri - d->r;
884 for (;;) {
885 d->r[i].p = NULL;
886 d->r[i].size = 0;
887 j = i;
888 for (;;) {
889 i = (i - 1) & mask;
890 if (d->r[i].p == NULL)
891 return;
892 r = hash(d->r[i].p) & mask;
893 if ((i <= r && r < j) || (r < j && j < i) ||
894 (j < i && i <= r))
895 continue;
896 d->r[j] = d->r[i];
897 STATS_INC(g_pool->delete_moves);
898 break;
899 }
900
901 }
902}
903
904/*
905 * Allocate a page of chunks
906 */
907static struct chunk_info *
908omalloc_make_chunks(struct dir_info *d, int bits)
909{
910 struct chunk_info *bp;
911 void *pp;
912 long i, k;
913
914 /* Allocate a new bucket */
915 pp = map(d, MALLOC_PAGESIZE, 0);
916 if (pp == MAP_FAILED)
917 return NULL;
918
919 bp = alloc_chunk_info(d);
920 if (bp == NULL) {
921 unmap(d, pp, MALLOC_PAGESIZE);
922 return NULL;
923 }
924
925 /* memory protect the page allocated in the malloc(0) case */
926 if (bits == 0) {
927 bp->size = 0;
928 bp->shift = 1;
929 i = MALLOC_MINSIZE - 1;
930 while (i >>= 1)
931 bp->shift++;
932 bp->total = bp->free = MALLOC_PAGESIZE >> bp->shift;
933 bp->page = pp;
934
935 k = mprotect(pp, MALLOC_PAGESIZE, PROT_NONE);
936 if (k < 0) {
937 unmap(d, pp, MALLOC_PAGESIZE);
938 LIST_INSERT_HEAD(&d->chunk_info_list, bp, entries);
939 return NULL;
940 }
941 } else {
942 bp->size = (1UL << bits);
943 bp->shift = bits;
944 bp->total = bp->free = MALLOC_PAGESIZE >> bits;
945 bp->page = pp;
946 }
947
948 /* set all valid bits in the bitmap */
949 k = bp->total;
950 i = 0;
951
952 /* Do a bunch at a time */
953 for (; (k - i) >= MALLOC_BITS; i += MALLOC_BITS)
954 bp->bits[i / MALLOC_BITS] = ~0UL;
955
956 for (; i < k; i++)
957 bp->bits[i / MALLOC_BITS] |= 1UL << (i % MALLOC_BITS);
958
959 LIST_INSERT_HEAD(&d->chunk_dir[bits], bp, entries);
960
961 bits++;
962 if ((uintptr_t)pp & bits)
963 wrterror("pp & bits");
964
965 insert(d, (void *)((uintptr_t)pp | bits), (uintptr_t)bp);
966 return bp;
967}
968
969
970/*
971 * Allocate a chunk
972 */
973static void *
974malloc_bytes(struct dir_info *d, size_t size)
975{
976 int i, j;
977 size_t k;
978 u_long u, *lp;
979 struct chunk_info *bp;
980
981 if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
982 d->canary1 != ~d->canary2)
983 wrterror("internal struct corrupt");
984 /* Don't bother with anything less than this */
985 /* unless we have a malloc(0) requests */
986 if (size != 0 && size < MALLOC_MINSIZE)
987 size = MALLOC_MINSIZE;
988
989 /* Find the right bucket */
990 if (size == 0)
991 j = 0;
992 else {
993 j = MALLOC_MINSHIFT;
994 i = (size - 1) >> (MALLOC_MINSHIFT - 1);
995 while (i >>= 1)
996 j++;
997 }
998
999 /* If it's empty, make a page more of that size chunks */
1000 if (LIST_EMPTY(&d->chunk_dir[j])) {
1001 bp = omalloc_make_chunks(d, j);
1002 if (bp == NULL)
1003 return NULL;
1004 } else
1005 bp = LIST_FIRST(&d->chunk_dir[j]);
1006
1007 if (bp->canary != d->canary1)
1008 wrterror("chunk info corrupted");
1009 /* Find first word of bitmap which isn't empty */
1010 for (lp = bp->bits; !*lp; lp++)
1011 /* EMPTY */;
1012
1013 /* Find that bit, and tweak it */
1014 u = 1;
1015 k = 0;
1016 while (!(*lp & u)) {
1017 u += u;
1018 k++;
1019 }
1020
1021 /* advance a random # of positions */
1022 i = getrnibble() % bp->free;
1023 while (i > 0) {
1024 u += u;
1025 k++;
1026 if (k >= MALLOC_BITS) {
1027 lp++;
1028 u = 1;
1029 k = 0;
1030 }
1031 if (lp - bp->bits > (bp->total - 1) / MALLOC_BITS) {
1032 wrterror("chunk overflow");
1033 errno = EFAULT;
1034 return (NULL);
1035 }
1036 if (*lp & u)
1037 i--;
1038 }
1039
1040 *lp ^= u;
1041
1042 /* If there are no more free, remove from free-list */
1043 if (!--bp->free)
1044 LIST_REMOVE(bp, entries);
1045
1046 /* Adjust to the real offset of that chunk */
1047 k += (lp - bp->bits) * MALLOC_BITS;
1048 k <<= bp->shift;
1049
1050 if (mopts.malloc_junk && bp->size > 0)
1051 memset((char *)bp->page + k, SOME_JUNK, bp->size);
1052 return ((char *)bp->page + k);
1053}
1054
1055
1056/*
1057 * Free a chunk, and possibly the page it's on, if the page becomes empty.
1058 */
1059static void
1060free_bytes(struct dir_info *d, struct region_info *r, void *ptr)
1061{
1062 struct chunk_head *mp;
1063 struct chunk_info *info;
1064 long i;
1065
1066 info = (struct chunk_info *)r->size;
1067 if (info->canary != d->canary1)
1068 wrterror("chunk info corrupted");
1069
1070 /* Find the chunk number on the page */
1071 i = ((uintptr_t)ptr & MALLOC_PAGEMASK) >> info->shift;
1072
1073 if ((uintptr_t)ptr & ((1UL << (info->shift)) - 1)) {
1074 wrterror("modified chunk-pointer");
1075 return;
1076 }
1077 if (info->bits[i / MALLOC_BITS] & (1UL << (i % MALLOC_BITS))) {
1078 wrterror("chunk is already free");
1079 return;
1080 }
1081
1082 info->bits[i / MALLOC_BITS] |= 1UL << (i % MALLOC_BITS);
1083 info->free++;
1084
1085 if (info->size != 0)
1086 mp = d->chunk_dir + info->shift;
1087 else
1088 mp = d->chunk_dir;
1089
1090 if (info->free == 1) {
1091 /* Page became non-full */
1092 LIST_INSERT_HEAD(mp, info, entries);
1093 return;
1094 }
1095 if (info->free != info->total)
1096 return;
1097
1098 LIST_REMOVE(info, entries);
1099
1100 if (info->size == 0 && !mopts.malloc_freeprot)
1101 mprotect(info->page, MALLOC_PAGESIZE, PROT_READ | PROT_WRITE);
1102 unmap(d, info->page, MALLOC_PAGESIZE);
1103
1104 delete(d, r);
1105 LIST_INSERT_HEAD(&d->chunk_info_list, info, entries);
1106}
1107
1108
1109
1110static void *
1111omalloc(size_t sz, int zero_fill)
1112{
1113 void *p;
1114 size_t psz;
1115
1116 if (sz > MALLOC_MAXCHUNK) {
1117 if (sz >= SIZE_MAX - mopts.malloc_guard - MALLOC_PAGESIZE) {
1118 errno = ENOMEM;
1119 return NULL;
1120 }
1121 sz += mopts.malloc_guard;
1122 psz = PAGEROUND(sz);
1123 p = map(g_pool, psz, zero_fill);
1124 if (p == MAP_FAILED) {
1125 errno = ENOMEM;
1126 return NULL;
1127 }
1128 if (insert(g_pool, p, sz)) {
1129 unmap(g_pool, p, psz);
1130 errno = ENOMEM;
1131 return NULL;
1132 }
1133 if (mopts.malloc_guard) {
1134 if (mprotect((char *)p + psz - mopts.malloc_guard,
1135 mopts.malloc_guard, PROT_NONE))
1136 wrterror("mprotect");
1137 malloc_guarded += mopts.malloc_guard;
1138 }
1139
1140 if (mopts.malloc_move &&
1141 sz - mopts.malloc_guard < MALLOC_PAGESIZE -
1142 MALLOC_LEEWAY) {
1143 /* fill whole allocation */
1144 if (mopts.malloc_junk)
1145 memset(p, SOME_JUNK, psz - mopts.malloc_guard);
1146 /* shift towards the end */
1147 p = ((char *)p) + ((MALLOC_PAGESIZE - MALLOC_LEEWAY -
1148 (sz - mopts.malloc_guard)) & ~(MALLOC_MINSIZE-1));
1149 /* fill zeros if needed and overwritten above */
1150 if (zero_fill && mopts.malloc_junk)
1151 memset(p, 0, sz - mopts.malloc_guard);
1152 } else {
1153 if (mopts.malloc_junk) {
1154 if (zero_fill)
1155 memset(p + sz - mopts.malloc_guard,
1156 SOME_JUNK, psz - sz);
1157 else
1158 memset(p, SOME_JUNK,
1159 psz - mopts.malloc_guard);
1160 }
1161 }
1162
1163 } else {
1164 /* takes care of SOME_JUNK */
1165 p = malloc_bytes(g_pool, sz);
1166 if (zero_fill && p != NULL && sz > 0)
1167 memset(p, 0, sz);
1168 }
1169
1170 return p;
1171}
1172
1173/*
1174 * Common function for handling recursion. Only
1175 * print the error message once, to avoid making the problem
1176 * potentially worse.
1177 */
1178static void
1179malloc_recurse(void)
1180{
1181 static int noprint;
1182
1183 if (noprint == 0) {
1184 noprint = 1;
1185 wrterror("recursive call");
1186 }
1187 malloc_active--;
1188 _MALLOC_UNLOCK();
1189 errno = EDEADLK;
1190}
1191
1192static int
1193malloc_init(void)
1194{
1195 if (omalloc_init(&g_pool)) {
1196 _MALLOC_UNLOCK();
1197 if (mopts.malloc_xmalloc)
1198 wrterror("out of memory");
1199 errno = ENOMEM;
1200 return -1;
1201 }
1202 return 0;
1203}
1204
1205void *
1206malloc(size_t size)
1207{
1208 void *r;
1209 int saved_errno = errno;
1210
1211 _MALLOC_LOCK();
1212 malloc_func = " in malloc():";
1213 if (g_pool == NULL) {
1214 if (malloc_init() != 0)
1215 return NULL;
1216 }
1217 if (malloc_active++) {
1218 malloc_recurse();
1219 return NULL;
1220 }
1221 r = omalloc(size, mopts.malloc_zero);
1222 malloc_active--;
1223 _MALLOC_UNLOCK();
1224 if (r == NULL && mopts.malloc_xmalloc) {
1225 wrterror("out of memory");
1226 errno = ENOMEM;
1227 }
1228 if (r != NULL)
1229 errno = saved_errno;
1230 return r;
1231}
1232
1233static void
1234ofree(void *p)
1235{
1236 struct region_info *r;
1237 size_t sz;
1238
1239 r = find(g_pool, p);
1240 if (r == NULL) {
1241 wrterror("bogus pointer (double free?)");
1242 return;
1243 }
1244 REALSIZE(sz, r);
1245 if (sz > MALLOC_MAXCHUNK) {
1246 if (sz - mopts.malloc_guard >= MALLOC_PAGESIZE -
1247 MALLOC_LEEWAY) {
1248 if (r->p != p) {
1249 wrterror("bogus pointer");
1250 return;
1251 }
1252 } else {
1253#if notyetbecause_of_realloc
1254 /* shifted towards the end */
1255 if (p != ((char *)r->p) + ((MALLOC_PAGESIZE -
1256 MALLOC_MINSIZE - sz - mopts.malloc_guard) &
1257 ~(MALLOC_MINSIZE-1))) {
1258 }
1259#endif
1260 p = r->p;
1261 }
1262 if (mopts.malloc_guard) {
1263 if (sz < mopts.malloc_guard)
1264 wrterror("guard size");
1265 if (!mopts.malloc_freeprot) {
1266 if (mprotect((char *)p + PAGEROUND(sz) -
1267 mopts.malloc_guard, mopts.malloc_guard,
1268 PROT_READ | PROT_WRITE))
1269 wrterror("mprotect");
1270 }
1271 malloc_guarded -= mopts.malloc_guard;
1272 }
1273 if (mopts.malloc_junk && !mopts.malloc_freeprot)
1274 memset(p, SOME_FREEJUNK,
1275 PAGEROUND(sz) - mopts.malloc_guard);
1276 unmap(g_pool, p, PAGEROUND(sz));
1277 delete(g_pool, r);
1278 } else {
1279 void *tmp;
1280 int i;
1281
1282 if (mopts.malloc_junk && sz > 0)
1283 memset(p, SOME_FREEJUNK, sz);
1284 if (!mopts.malloc_freeprot) {
1285 i = getrnibble();
1286 tmp = p;
1287 p = g_pool->delayed_chunks[i];
1288 g_pool->delayed_chunks[i] = tmp;
1289 }
1290 if (p != NULL) {
1291 r = find(g_pool, p);
1292 if (r == NULL) {
1293 wrterror("bogus pointer (double free?)");
1294 return;
1295 }
1296 free_bytes(g_pool, r, p);
1297 }
1298 }
1299}
1300
1301void
1302free(void *ptr)
1303{
1304 int saved_errno = errno;
1305
1306 /* This is legal. */
1307 if (ptr == NULL)
1308 return;
1309
1310 _MALLOC_LOCK();
1311 malloc_func = " in free():";
1312 if (g_pool == NULL) {
1313 _MALLOC_UNLOCK();
1314 wrterror("free() called before allocation");
1315 return;
1316 }
1317 if (malloc_active++) {
1318 malloc_recurse();
1319 return;
1320 }
1321 ofree(ptr);
1322 malloc_active--;
1323 _MALLOC_UNLOCK();
1324 errno = saved_errno;
1325}
1326
1327
1328static void *
1329orealloc(void *p, size_t newsz)
1330{
1331 struct region_info *r;
1332 size_t oldsz, goldsz, gnewsz;
1333 void *q;
1334
1335 if (p == NULL)
1336 return omalloc(newsz, 0);
1337
1338 r = find(g_pool, p);
1339 if (r == NULL) {
1340 wrterror("bogus pointer (double free?)");
1341 return NULL;
1342 }
1343 if (newsz >= SIZE_MAX - mopts.malloc_guard - MALLOC_PAGESIZE) {
1344 errno = ENOMEM;
1345 return NULL;
1346 }
1347
1348 REALSIZE(oldsz, r);
1349 goldsz = oldsz;
1350 if (oldsz > MALLOC_MAXCHUNK) {
1351 if (oldsz < mopts.malloc_guard)
1352 wrterror("guard size");
1353 oldsz -= mopts.malloc_guard;
1354 }
1355
1356 gnewsz = newsz;
1357 if (gnewsz > MALLOC_MAXCHUNK)
1358 gnewsz += mopts.malloc_guard;
1359
1360 if (newsz > MALLOC_MAXCHUNK && oldsz > MALLOC_MAXCHUNK && p == r->p &&
1361 !mopts.malloc_realloc) {
1362 size_t roldsz = PAGEROUND(goldsz);
1363 size_t rnewsz = PAGEROUND(gnewsz);
1364
1365 if (rnewsz > roldsz) {
1366 if (!mopts.malloc_guard) {
1367 STATS_INC(g_pool->cheap_realloc_tries);
1368 zapcacheregion(g_pool, p + roldsz);
1369 q = MMAPA(p + roldsz, rnewsz - roldsz);
1370 if (q == p + roldsz) {
1371 malloc_used += rnewsz - roldsz;
1372 if (mopts.malloc_junk)
1373 memset(q, SOME_JUNK,
1374 rnewsz - roldsz);
1375 r->size = newsz;
1376 STATS_INC(g_pool->cheap_reallocs);
1377 return p;
1378 } else if (q != MAP_FAILED)
1379 munmap(q, rnewsz - roldsz);
1380 }
1381 } else if (rnewsz < roldsz) {
1382 if (mopts.malloc_guard) {
1383 if (mprotect((char *)p + roldsz -
1384 mopts.malloc_guard, mopts.malloc_guard,
1385 PROT_READ | PROT_WRITE))
1386 wrterror("mprotect");
1387 if (mprotect((char *)p + rnewsz -
1388 mopts.malloc_guard, mopts.malloc_guard,
1389 PROT_NONE))
1390 wrterror("mprotect");
1391 }
1392 unmap(g_pool, (char *)p + rnewsz, roldsz - rnewsz);
1393 r->size = gnewsz;
1394 return p;
1395 } else {
1396 if (newsz > oldsz && mopts.malloc_junk)
1397 memset((char *)p + newsz, SOME_JUNK,
1398 rnewsz - mopts.malloc_guard - newsz);
1399 r->size = gnewsz;
1400 return p;
1401 }
1402 }
1403 if (newsz <= oldsz && newsz > oldsz / 2 && !mopts.malloc_realloc) {
1404 if (mopts.malloc_junk && newsz > 0)
1405 memset((char *)p + newsz, SOME_JUNK, oldsz - newsz);
1406 return p;
1407 } else if (newsz != oldsz || mopts.malloc_realloc) {
1408 q = omalloc(newsz, 0);
1409 if (q == NULL)
1410 return NULL;
1411 if (newsz != 0 && oldsz != 0)
1412 memcpy(q, p, oldsz < newsz ? oldsz : newsz);
1413 ofree(p);
1414 return q;
1415 } else
1416 return p;
1417}
1418
1419void *
1420realloc(void *ptr, size_t size)
1421{
1422 void *r;
1423 int saved_errno = errno;
1424
1425 _MALLOC_LOCK();
1426 malloc_func = " in realloc():";
1427 if (g_pool == NULL) {
1428 if (malloc_init() != 0)
1429 return NULL;
1430 }
1431 if (malloc_active++) {
1432 malloc_recurse();
1433 return NULL;
1434 }
1435 r = orealloc(ptr, size);
1436
1437 malloc_active--;
1438 _MALLOC_UNLOCK();
1439 if (r == NULL && mopts.malloc_xmalloc) {
1440 wrterror("out of memory");
1441 errno = ENOMEM;
1442 }
1443 if (r != NULL)
1444 errno = saved_errno;
1445 return r;
1446}
1447
1448
1449#define MUL_NO_OVERFLOW (1UL << (sizeof(size_t) * 4))
1450
1451void *
1452calloc(size_t nmemb, size_t size)
1453{
1454 void *r;
1455 int saved_errno = errno;
1456
1457 _MALLOC_LOCK();
1458 malloc_func = " in calloc():";
1459 if (g_pool == NULL) {
1460 if (malloc_init() != 0)
1461 return NULL;
1462 }
1463 if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
1464 nmemb > 0 && SIZE_MAX / nmemb < size) {
1465 _MALLOC_UNLOCK();
1466 if (mopts.malloc_xmalloc)
1467 wrterror("out of memory");
1468 errno = ENOMEM;
1469 return NULL;
1470 }
1471
1472 if (malloc_active++) {
1473 malloc_recurse();
1474 return NULL;
1475 }
1476
1477 size *= nmemb;
1478 r = omalloc(size, 1);
1479
1480 malloc_active--;
1481 _MALLOC_UNLOCK();
1482 if (r == NULL && mopts.malloc_xmalloc) {
1483 wrterror("out of memory");
1484 errno = ENOMEM;
1485 }
1486 if (r != NULL)
1487 errno = saved_errno;
1488 return r;
1489}
1490
1491int
1492posix_memalign(void **memptr, size_t alignment, size_t size)
1493{
1494 void *result;
1495
1496 /* Make sure that alignment is a large enough power of 2. */
1497 if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *) ||
1498 alignment > MALLOC_PAGESIZE)
1499 return EINVAL;
1500
1501 /*
1502 * max(size, alignment) is enough to assure the requested alignment,
1503 * since the allocator always allocates power-of-two blocks.
1504 */
1505 if (size < alignment)
1506 size = alignment;
1507 result = malloc(size);
1508
1509 if (result == NULL)
1510 return ENOMEM;
1511
1512 *memptr = result;
1513 return 0;
1514}
1515
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-11-07 00:57:23 +0000