17 #include "kmp_wrapper_malloc.h" 25 typedef int (*bget_compact_t)(size_t, int);
26 typedef void *(*bget_acquire_t)(size_t);
27 typedef void (*bget_release_t)(
void *);
32 # if KMP_ARCH_X86 || KMP_ARCH_ARM 33 typedef kmp_int32 bufsize;
35 typedef kmp_int64 bufsize;
38 typedef ssize_t bufsize;
43 typedef enum bget_mode {
50 static void bpool( kmp_info_t *th,
void *buffer, bufsize len);
51 static void *bget( kmp_info_t *th, bufsize size);
52 static void *bgetz( kmp_info_t *th, bufsize size);
53 static void *bgetr( kmp_info_t *th,
void *buffer, bufsize newsize);
54 static void brel( kmp_info_t *th,
void *buf);
55 static void bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr );
58 static void bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
long *nget,
long *nrel);
59 static void bstatse( kmp_info_t *th, bufsize *pool_incr,
long *npool,
long *npget,
long *nprel,
long *ndget,
long *ndrel);
60 static void bufdump( kmp_info_t *th,
void *buf);
61 static void bpoold( kmp_info_t *th,
void *pool,
int dumpalloc,
int dumpfree);
62 static int bpoolv( kmp_info_t *th,
void *pool);
75 #if KMP_ARCH_X86 || !KMP_HAVE_QUAD 78 #define AlignType double 83 #define AlignType _Quad 124 static bufsize bget_bin_size[ ] = {
148 #define MAX_BGET_BINS (int)(sizeof(bget_bin_size) / sizeof(bufsize)) 157 typedef struct qlinks {
158 struct bfhead *flink;
159 struct bfhead *blink;
164 typedef struct bhead2 {
175 typedef union bhead {
176 KMP_ALIGN( SizeQuant )
178 char b_pad[ sizeof(bhead2_t) + (SizeQuant - (sizeof(bhead2_t) % SizeQuant)) ];
181 #define BH(p) ((bhead_t *) (p)) 185 typedef struct bdhead
190 #define BDH(p) ((bdhead_t *) (p)) 194 typedef struct bfhead {
198 #define BFH(p) ((bfhead_t *) (p)) 200 typedef struct thr_data {
201 bfhead_t freelist[ MAX_BGET_BINS ];
206 long numpget, numprel;
207 long numdget, numdrel;
211 bget_compact_t compfcn;
212 bget_acquire_t acqfcn;
213 bget_release_t relfcn;
224 bfhead_t * last_pool;
229 #define QLSize (sizeof(qlinks_t)) 230 #define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize) 231 #define MaxSize (bufsize)( ~ ( ( (bufsize)( 1 ) << ( sizeof( bufsize ) * CHAR_BIT - 1 ) ) | ( SizeQuant - 1 ) ) ) 238 #define ESent ((bufsize) (-(((((bufsize)1)<<((int)sizeof(bufsize)*8-2))-1)*2)-2)) 245 bget_get_bin( bufsize size )
248 int lo = 0, hi = MAX_BGET_BINS - 1;
250 KMP_DEBUG_ASSERT( size > 0 );
252 while ( (hi - lo) > 1 ) {
253 int mid = (lo + hi) >> 1;
254 if (size < bget_bin_size[ mid ])
260 KMP_DEBUG_ASSERT( (lo >= 0) && (lo < MAX_BGET_BINS) );
266 set_thr_data( kmp_info_t *th )
273 ( ! th->th.th_local.bget_data ) ? __kmp_allocate(
sizeof( *data ) ) : th->th.th_local.bget_data
276 memset( data,
'\0',
sizeof( *data ) );
278 for (i = 0; i < MAX_BGET_BINS; ++i) {
279 data->freelist[ i ].ql.flink = & data->freelist[ i ];
280 data->freelist[ i ].ql.blink = & data->freelist[ i ];
283 th->th.th_local.bget_data = data;
284 th->th.th_local.bget_list = 0;
285 #if ! USE_CMP_XCHG_FOR_BGET 286 #ifdef USE_QUEUING_LOCK_FOR_BGET 287 __kmp_init_lock( & th->th.th_local.bget_lock );
289 __kmp_init_bootstrap_lock( & th->th.th_local.bget_lock );
295 get_thr_data( kmp_info_t *th )
299 data = (thr_data_t *) th->th.th_local.bget_data;
301 KMP_DEBUG_ASSERT( data != 0 );
310 __kmp_bget_validate_queue( kmp_info_t *th )
314 void *p = (
void *) th->th.th_local.bget_list;
317 bfhead_t *b = BFH(((
char *) p) -
sizeof(bhead_t));
319 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
320 p = (
void *) b->ql.flink;
329 __kmp_bget_dequeue( kmp_info_t *th )
331 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list);
334 #if USE_CMP_XCHG_FOR_BGET 336 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
337 while ( ! KMP_COMPARE_AND_STORE_PTR(
338 & th->th.th_local.bget_list, old_value, NULL ) )
341 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
343 p = (
void *) old_value;
346 #ifdef USE_QUEUING_LOCK_FOR_BGET 347 __kmp_acquire_lock( & th->th.th_local.bget_lock,
348 __kmp_gtid_from_thread(th) );
350 __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
353 p = (
void *) th->th.th_local.bget_list;
354 th->th.th_local.bget_list = 0;
356 #ifdef USE_QUEUING_LOCK_FOR_BGET
357 __kmp_release_lock( & th->th.th_local.bget_lock,
358 __kmp_gtid_from_thread(th) );
360 __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
368 bfhead_t *b = BFH(((
char *) p) -
sizeof(bhead_t));
370 KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
371 KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
373 KMP_DEBUG_ASSERT( b->ql.blink == 0 );
375 p = (
void *) b->ql.flink;
385 __kmp_bget_enqueue( kmp_info_t *th,
void *buf
386 #ifdef USE_QUEUING_LOCK_FOR_BGET
391 bfhead_t *b = BFH(((
char *) buf) -
sizeof(bhead_t));
393 KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
394 KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
399 KC_TRACE( 10, (
"__kmp_bget_enqueue: moving buffer to T#%d list\n",
400 __kmp_gtid_from_thread( th ) ) );
402 #if USE_CMP_XCHG_FOR_BGET 404 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list);
407 b->ql.flink = BFH( old_value );
409 while ( ! KMP_COMPARE_AND_STORE_PTR(
410 & th->th.th_local.bget_list, old_value, buf ) )
413 old_value = TCR_PTR(th->th.th_local.bget_list);
416 b->ql.flink = BFH( old_value );
420 # ifdef USE_QUEUING_LOCK_FOR_BGET 421 __kmp_acquire_lock( & th->th.th_local.bget_lock, rel_gtid );
423 __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
426 b->ql.flink = BFH( th->th.th_local.bget_list );
427 th->th.th_local.bget_list = (
void *) buf;
429 # ifdef USE_QUEUING_LOCK_FOR_BGET 430 __kmp_release_lock( & th->th.th_local.bget_lock, rel_gtid );
432 __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
440 __kmp_bget_insert_into_freelist( thr_data_t *thr, bfhead_t *b )
444 KMP_DEBUG_ASSERT( ((
size_t)b ) % SizeQuant == 0 );
445 KMP_DEBUG_ASSERT( b->bh.bb.bsize % SizeQuant == 0 );
447 bin = bget_get_bin( b->bh.bb.bsize );
449 KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.blink->ql.flink == &thr->freelist[ bin ]);
450 KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.flink->ql.blink == &thr->freelist[ bin ]);
452 b->ql.flink = &thr->freelist[ bin ];
453 b->ql.blink = thr->freelist[ bin ].ql.blink;
455 thr->freelist[ bin ].ql.blink = b;
456 b->ql.blink->ql.flink = b;
462 __kmp_bget_remove_from_freelist( bfhead_t *b )
464 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b);
465 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b);
467 b->ql.blink->ql.flink = b->ql.flink;
468 b->ql.flink->ql.blink = b->ql.blink;
476 bcheck( kmp_info_t *th, bufsize *max_free, bufsize *total_free )
478 thr_data_t *thr = get_thr_data( th );
481 *total_free = *max_free = 0;
483 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
486 best = &thr->freelist[ bin ];
489 while (b != &thr->freelist[ bin ]) {
490 *total_free += (b->bh.bb.bsize -
sizeof( bhead_t ));
491 if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize))
498 if (*max_free < best->bh.bb.bsize)
499 *max_free = best->bh.bb.bsize;
502 if (*max_free > (bufsize)
sizeof( bhead_t ))
503 *max_free -=
sizeof( bhead_t );
511 bget( kmp_info_t *th, bufsize requested_size )
513 thr_data_t *thr = get_thr_data( th );
514 bufsize size = requested_size;
522 if ( size < 0 || size +
sizeof( bhead_t ) > MaxSize ) {
526 __kmp_bget_dequeue( th );
528 if (size < (bufsize)SizeQ) {
531 #if defined( SizeQuant ) && ( SizeQuant > 1 ) 532 size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
535 size +=
sizeof(bhead_t);
537 KMP_DEBUG_ASSERT( size >= 0 );
538 KMP_DEBUG_ASSERT( size % SizeQuant == 0 );
540 use_blink = ( thr->mode == bget_mode_lifo );
549 for (bin = bget_get_bin( size ); bin < MAX_BGET_BINS; ++bin) {
551 b = ( use_blink ? thr->freelist[ bin ].ql.blink : thr->freelist[ bin ].ql.flink );
553 if (thr->mode == bget_mode_best) {
554 best = &thr->freelist[ bin ];
559 while (b != &thr->freelist[ bin ]) {
560 if (b->bh.bb.bsize >= (bufsize) size) {
561 if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize)) {
567 b = ( use_blink ? b->ql.blink : b->ql.flink );
572 while (b != &thr->freelist[ bin ]) {
573 if ((bufsize) b->bh.bb.bsize >= (bufsize) size) {
584 if ((b->bh.bb.bsize - (bufsize) size) > (bufsize)(SizeQ + (
sizeof(bhead_t)))) {
587 ba = BH(((
char *) b) + (b->bh.bb.bsize - (bufsize) size));
588 bn = BH(((
char *) ba) + size);
590 KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize);
593 b->bh.bb.bsize -= (bufsize) size;
596 ba->bb.prevfree = b->bh.bb.bsize;
599 ba->bb.bsize = -size;
602 TCW_PTR(ba->bb.bthr, th);
607 __kmp_bget_remove_from_freelist( b );
608 __kmp_bget_insert_into_freelist( thr, b );
610 thr->totalloc += (size_t) size;
613 buf = (
void *) ((((
char *) ba) +
sizeof(bhead_t)));
614 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
619 ba = BH(((
char *) b) + b->bh.bb.bsize);
621 KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize);
626 __kmp_bget_remove_from_freelist( b );
628 thr->totalloc += (size_t) b->bh.bb.bsize;
632 b->bh.bb.bsize = -(b->bh.bb.bsize);
635 TCW_PTR(ba->bb.bthr, th);
641 buf = (
void *) &(b->ql);
642 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
648 b = ( use_blink ? b->ql.blink : b->ql.flink );
656 if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) {
665 if (thr->acqfcn != 0) {
666 if (size > (bufsize) (thr->exp_incr -
sizeof(bhead_t))) {
673 size +=
sizeof(bdhead_t) -
sizeof(bhead_t);
675 KE_TRACE( 10, (
"%%%%%% MALLOC( %d )\n", (
int) size ) );
678 bdh = BDH((*thr->acqfcn)((bufsize) size));
683 bdh->bh.bb.bsize = 0;
686 TCW_PTR(bdh->bh.bb.bthr, th);
688 bdh->bh.bb.prevfree = 0;
691 thr->totalloc += (size_t) size;
695 buf = (
void *) (bdh + 1);
696 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
706 KE_TRACE( 10, (
"%%%%%% MALLOCB( %d )\n", (
int) thr->exp_incr ) );
709 newpool = (*thr->acqfcn)((bufsize) thr->exp_incr);
710 KMP_DEBUG_ASSERT( ((
size_t)newpool) % SizeQuant == 0 );
711 if (newpool != NULL) {
712 bpool( th, newpool, thr->exp_incr);
713 buf = bget( th, requested_size);
729 bgetz( kmp_info_t *th, bufsize size )
731 char *buf = (
char *) bget( th, size);
737 b = BH(buf -
sizeof(bhead_t));
738 rsize = -(b->bb.bsize);
742 bd = BDH(buf -
sizeof(bdhead_t));
743 rsize = bd->tsize - (bufsize)
sizeof(bdhead_t);
745 rsize -=
sizeof(bhead_t);
748 KMP_DEBUG_ASSERT(rsize >= size);
750 (void) memset(buf, 0, (bufsize) rsize);
752 return ((
void *) buf);
761 bgetr( kmp_info_t *th,
void *buf, bufsize size)
767 nbuf = bget( th, size );
768 if ( nbuf == NULL ) {
774 b = BH(((
char *) buf) -
sizeof(bhead_t));
775 osize = -b->bb.bsize;
780 bd = BDH(((
char *) buf) -
sizeof(bdhead_t));
781 osize = bd->tsize - (bufsize)
sizeof(bdhead_t);
783 osize -=
sizeof(bhead_t);
786 KMP_DEBUG_ASSERT(osize > 0);
788 (void) KMP_MEMCPY((
char *) nbuf, (
char *) buf,
789 (
size_t) ((size < osize) ? size : osize));
798 brel( kmp_info_t *th,
void *buf )
800 thr_data_t *thr = get_thr_data( th );
804 KMP_DEBUG_ASSERT(buf != NULL);
805 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
807 b = BFH(((
char *) buf) -
sizeof(bhead_t));
809 if (b->bh.bb.bsize == 0) {
812 bdh = BDH(((
char *) buf) -
sizeof(bdhead_t));
813 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
815 thr->totalloc -= (size_t) bdh->tsize;
820 (
void) memset((
char *) buf, 0x55,
821 (
size_t) (bdh->tsize -
sizeof(bdhead_t)));
824 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) bdh ) );
826 KMP_DEBUG_ASSERT( thr->relfcn != 0 );
827 (*thr->relfcn)((
void *) bdh);
831 bth = (kmp_info_t *)( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 );
834 __kmp_bget_enqueue( bth, buf
835 #ifdef USE_QUEUING_LOCK_FOR_BGET
836 , __kmp_gtid_from_thread( th )
845 if (b->bh.bb.bsize >= 0) {
848 KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0);
853 KMP_DEBUG_ASSERT(BH((
char *) b - b->bh.bb.bsize)->bb.prevfree == 0);
857 thr->totalloc += (size_t) b->bh.bb.bsize;
862 if (b->bh.bb.prevfree != 0) {
869 register bufsize size = b->bh.bb.bsize;
872 KMP_DEBUG_ASSERT(BH((
char *) b - b->bh.bb.prevfree)->bb.bsize == b->bh.bb.prevfree);
873 b = BFH(((
char *) b) - b->bh.bb.prevfree);
874 b->bh.bb.bsize -= size;
877 __kmp_bget_remove_from_freelist( b );
884 b->bh.bb.bsize = -b->bh.bb.bsize;
888 __kmp_bget_insert_into_freelist( thr, b );
896 bn = BFH(((
char *) b) + b->bh.bb.bsize);
897 if (bn->bh.bb.bsize > 0) {
902 KMP_DEBUG_ASSERT(BH((
char *) bn + bn->bh.bb.bsize)->bb.prevfree == bn->bh.bb.bsize);
904 __kmp_bget_remove_from_freelist( bn );
906 b->bh.bb.bsize += bn->bh.bb.bsize;
910 __kmp_bget_remove_from_freelist( b );
911 __kmp_bget_insert_into_freelist( thr, b );
920 bn = BFH(((
char *) b) + b->bh.bb.bsize);
923 (void) memset(((
char *) b) +
sizeof(bfhead_t), 0x55,
924 (
size_t) (b->bh.bb.bsize -
sizeof(bfhead_t)));
926 KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0);
931 bn->bh.bb.prevfree = b->bh.bb.bsize;
938 if (thr->relfcn != 0 &&
939 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t)))
942 if (thr->numpblk != 1) {
945 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
946 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
947 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
950 __kmp_bget_remove_from_freelist( b );
952 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) b ) );
958 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
961 if (thr->last_pool == b) thr->last_pool = 0;
973 bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr)
975 thr_data_t *thr = get_thr_data( th );
977 thr->compfcn = compact;
978 thr->acqfcn = acquire;
979 thr->relfcn = release;
980 thr->exp_incr = pool_incr;
986 bpool( kmp_info_t *th,
void *buf, bufsize len)
989 thr_data_t *thr = get_thr_data( th );
990 bfhead_t *b = BFH(buf);
993 __kmp_bget_dequeue( th );
996 len &= ~(SizeQuant - 1);
998 if (thr->pool_len == 0) {
1000 }
else if (len != thr->pool_len) {
1006 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1013 KMP_DEBUG_ASSERT(len -
sizeof(bhead_t) <= -((bufsize) ESent + 1));
1019 b->bh.bb.prevfree = 0;
1029 len -=
sizeof(bhead_t);
1030 b->bh.bb.bsize = (bufsize) len;
1032 TCW_PTR( b->bh.bb.bthr, (kmp_info_t*)((kmp_uintptr_t)th | 1) );
1035 __kmp_bget_insert_into_freelist( thr, b );
1038 (void) memset(((
char *) b) +
sizeof(bfhead_t), 0x55,
1039 (
size_t) (len -
sizeof(bfhead_t)));
1041 bn = BH(((
char *) b) + len);
1042 bn->bb.prevfree = (bufsize) len;
1044 KMP_DEBUG_ASSERT( (~0) == -1 && (bn != 0) );
1046 bn->bb.bsize = ESent;
1054 bfreed( kmp_info_t *th )
1056 int bin = 0, count = 0;
1057 int gtid = __kmp_gtid_from_thread( th );
1058 thr_data_t *thr = get_thr_data( th );
1061 __kmp_printf_no_lock(
"__kmp_printpool: T#%d total=%" KMP_UINT64_SPEC
" get=%" KMP_INT64_SPEC
" rel=%" \
1062 KMP_INT64_SPEC
" pblk=%" KMP_INT64_SPEC
" pget=%" KMP_INT64_SPEC
" prel=%" KMP_INT64_SPEC \
1063 " dget=%" KMP_INT64_SPEC
" drel=%" KMP_INT64_SPEC
"\n",
1064 gtid, (kmp_uint64) thr->totalloc,
1065 (kmp_int64) thr->numget, (kmp_int64) thr->numrel,
1066 (kmp_int64) thr->numpblk,
1067 (kmp_int64) thr->numpget, (kmp_int64) thr->numprel,
1068 (kmp_int64) thr->numdget, (kmp_int64) thr->numdrel );
1071 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1074 for (b = thr->freelist[ bin ].ql.flink; b != &thr->freelist[ bin ]; b = b->ql.flink) {
1075 bufsize bs = b->bh.bb.bsize;
1077 KMP_DEBUG_ASSERT( b->ql.blink->ql.flink == b );
1078 KMP_DEBUG_ASSERT( b->ql.flink->ql.blink == b );
1079 KMP_DEBUG_ASSERT( bs > 0 );
1083 __kmp_printf_no_lock(
"__kmp_printpool: T#%d Free block: 0x%p size %6ld bytes.\n", gtid, b, (
long) bs );
1086 char *lerr = ((
char *) b) +
sizeof(bfhead_t);
1087 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) || (memcmp(lerr, lerr + 1, (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1088 __kmp_printf_no_lock(
"__kmp_printpool: T#%d (Contents of above free block have been overstored.)\n", gtid );
1096 __kmp_printf_no_lock(
"__kmp_printpool: T#%d No free blocks\n", gtid );
1108 bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
long *nget,
long *nrel)
1111 thr_data_t *thr = get_thr_data( th );
1113 *nget = thr->numget;
1114 *nrel = thr->numrel;
1115 *curalloc = (bufsize) thr->totalloc;
1119 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1120 bfhead_t *b = thr->freelist[ bin ].ql.flink;
1122 while (b != &thr->freelist[ bin ]) {
1123 KMP_DEBUG_ASSERT(b->bh.bb.bsize > 0);
1124 *totfree += b->bh.bb.bsize;
1125 if (b->bh.bb.bsize > *maxfree) {
1126 *maxfree = b->bh.bb.bsize;
1136 bstatse( kmp_info_t *th, bufsize *pool_incr,
long *npool,
long *npget,
long *nprel,
long *ndget,
long *ndrel)
1138 thr_data_t *thr = get_thr_data( th );
1140 *pool_incr = (thr->pool_len < 0) ? -thr->exp_incr : thr->exp_incr;
1141 *npool = thr->numpblk;
1142 *npget = thr->numpget;
1143 *nprel = thr->numprel;
1144 *ndget = thr->numdget;
1145 *ndrel = thr->numdrel;
1155 bufdump( kmp_info_t *th,
void *buf )
1158 unsigned char *bdump;
1161 b = BFH(((
char *) buf) -
sizeof(bhead_t));
1162 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
1163 if (b->bh.bb.bsize < 0) {
1164 bdump = (
unsigned char *) buf;
1165 bdlen = (-b->bh.bb.bsize) - (bufsize)
sizeof(bhead_t);
1167 bdump = (
unsigned char *) (((
char *) b) +
sizeof(bfhead_t));
1168 bdlen = b->bh.bb.bsize - (bufsize)
sizeof(bfhead_t);
1174 char bhex[50], bascii[20];
1180 for (i = 0; i < l; i++) {
1181 (void) KMP_SNPRINTF(bhex + i * 3,
sizeof(bhex),
"%02X ", bdump[i]);
1182 if (bdump[i] > 0x20 && bdump[i] < 0x7F)
1183 bascii[ i ] = bdump[ i ];
1188 (void) __kmp_printf_no_lock(
"%-48s %s\n", bhex, bascii);
1191 while ((bdlen > 16) && (memcmp((
char *) (bdump - 16),
1192 (
char *) bdump, 16) == 0)) {
1198 (void) __kmp_printf_no_lock(
1199 " (%d lines [%d bytes] identical to above line skipped)\n",
1201 }
else if (dupes == 1) {
1215 bpoold( kmp_info_t *th,
void *buf,
int dumpalloc,
int dumpfree)
1217 bfhead_t *b = BFH( (
char*)buf -
sizeof(bhead_t));
1219 while (b->bh.bb.bsize != ESent) {
1220 bufsize bs = b->bh.bb.bsize;
1224 (void) __kmp_printf_no_lock(
"Allocated buffer: size %6ld bytes.\n", (
long) bs);
1226 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1229 const char *lerr =
"";
1231 KMP_DEBUG_ASSERT(bs > 0);
1232 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1233 lerr =
" (Bad free list links)";
1235 (void) __kmp_printf_no_lock(
"Free block: size %6ld bytes.%s\n",
1238 lerr = ((
char *) b) +
sizeof(bfhead_t);
1239 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) ||
1240 (memcmp(lerr, lerr + 1,
1241 (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1242 (void) __kmp_printf_no_lock(
1243 "(Contents of above free block have been overstored.)\n");
1244 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1248 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1251 b = BFH(((
char *) b) + bs);
1258 bpoolv( kmp_info_t *th,
void *buf )
1260 bfhead_t *b = BFH(buf);
1262 while (b->bh.bb.bsize != ESent) {
1263 bufsize bs = b->bh.bb.bsize;
1272 KMP_DEBUG_ASSERT(bs > 0);
1276 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1277 (void) __kmp_printf_no_lock(
"Free block: size %6ld bytes. (Bad free list links)\n",
1279 KMP_DEBUG_ASSERT(0);
1283 lerr = ((
char *) b) +
sizeof(bfhead_t);
1284 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) ||
1285 (memcmp(lerr, lerr + 1,
1286 (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1287 (void) __kmp_printf_no_lock(
1288 "(Contents of above free block have been overstored.)\n");
1289 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1290 KMP_DEBUG_ASSERT(0);
1295 b = BFH(((
char *) b) + bs);
1305 __kmp_initialize_bget( kmp_info_t *th )
1307 KMP_DEBUG_ASSERT( SizeQuant >=
sizeof(
void * ) && (th != 0) );
1311 bectl( th, (bget_compact_t) 0, (bget_acquire_t) malloc, (bget_release_t) free,
1312 (bufsize) __kmp_malloc_pool_incr );
1316 __kmp_finalize_bget( kmp_info_t *th )
1321 KMP_DEBUG_ASSERT( th != 0 );
1324 thr = (thr_data_t *) th->th.th_local.bget_data;
1325 KMP_DEBUG_ASSERT( thr != NULL );
1334 if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 &&
1335 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t)))
1337 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
1338 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
1339 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
1342 __kmp_bget_remove_from_freelist( b );
1344 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) b ) );
1349 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1354 if ( th->th.th_local.bget_data != NULL ) {
1355 __kmp_free( th->th.th_local.bget_data );
1356 th->th.th_local.bget_data = NULL;
1361 kmpc_set_poolsize(
size_t size )
1363 bectl( __kmp_get_thread(), (bget_compact_t) 0, (bget_acquire_t) malloc,
1364 (bget_release_t) free, (bufsize) size );
1368 kmpc_get_poolsize(
void )
1372 p = get_thr_data( __kmp_get_thread() );
1378 kmpc_set_poolmode(
int mode )
1382 if (mode == bget_mode_fifo || mode == bget_mode_lifo || mode == bget_mode_best) {
1383 p = get_thr_data( __kmp_get_thread() );
1384 p->mode = (bget_mode_t) mode;
1389 kmpc_get_poolmode(
void )
1393 p = get_thr_data( __kmp_get_thread() );
1399 kmpc_get_poolstat(
size_t *maxmem,
size_t *allmem )
1401 kmp_info_t *th = __kmp_get_thread();
1404 __kmp_bget_dequeue( th );
1406 bcheck( th, &a, &b );
1413 kmpc_poolprint(
void )
1415 kmp_info_t *th = __kmp_get_thread();
1417 __kmp_bget_dequeue( th );
1422 #endif // #if KMP_USE_BGET 1427 kmpc_malloc(
size_t size )
1430 ptr = bget( __kmp_entry_thread(), (bufsize) size );
1436 kmpc_calloc(
size_t nelem,
size_t elsize )
1439 ptr = bgetz( __kmp_entry_thread(), (bufsize) (nelem * elsize) );
1445 kmpc_realloc(
void * ptr,
size_t size )
1447 void * result = NULL;
1449 if ( ptr == NULL ) {
1451 result = bget( __kmp_entry_thread(), (bufsize) size );
1452 }
else if ( size == 0 ) {
1456 brel( __kmp_get_thread(), ptr );
1458 result = bgetr( __kmp_entry_thread(), ptr, (bufsize) size );
1467 kmpc_free(
void * ptr )
1469 if ( ! __kmp_init_serial ) {
1472 if ( ptr != NULL ) {
1473 kmp_info_t *th = __kmp_get_thread();
1474 __kmp_bget_dequeue( th );
1483 ___kmp_thread_malloc( kmp_info_t *th,
size_t size KMP_SRC_LOC_DECL )
1487 "-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n",
1492 ptr = bget( th, (bufsize) size );
1493 KE_TRACE( 30, (
"<- __kmp_thread_malloc() returns %p\n", ptr ) );
1498 ___kmp_thread_calloc( kmp_info_t *th,
size_t nelem,
size_t elsize KMP_SRC_LOC_DECL )
1502 "-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n",
1508 ptr = bgetz( th, (bufsize) (nelem * elsize) );
1509 KE_TRACE( 30, (
"<- __kmp_thread_calloc() returns %p\n", ptr ) );
1514 ___kmp_thread_realloc( kmp_info_t *th,
void *ptr,
size_t size KMP_SRC_LOC_DECL )
1517 "-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n",
1523 ptr = bgetr( th, ptr, (bufsize) size );
1524 KE_TRACE( 30, (
"<- __kmp_thread_realloc() returns %p\n", ptr ) );
1529 ___kmp_thread_free( kmp_info_t *th,
void *ptr KMP_SRC_LOC_DECL )
1532 "-> __kmp_thread_free( %p, %p ) called from %s:%d\n",
1537 if ( ptr != NULL ) {
1538 __kmp_bget_dequeue( th );
1541 KE_TRACE( 30, (
"<- __kmp_thread_free()\n" ) );
1552 struct kmp_mem_descr {
1553 void * ptr_allocated;
1554 size_t size_allocated;
1556 size_t size_aligned;
1558 typedef struct kmp_mem_descr kmp_mem_descr_t;
1567 ___kmp_allocate_align(
size_t size,
size_t alignment KMP_SRC_LOC_DECL )
1586 kmp_mem_descr_t descr;
1587 kmp_uintptr_t addr_allocated;
1588 kmp_uintptr_t addr_aligned;
1589 kmp_uintptr_t addr_descr;
1592 "-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n",
1598 KMP_DEBUG_ASSERT( alignment < 32 * 1024 );
1599 KMP_DEBUG_ASSERT(
sizeof(
void * ) <=
sizeof( kmp_uintptr_t ) );
1602 descr.size_aligned = size;
1603 descr.size_allocated = descr.size_aligned +
sizeof( kmp_mem_descr_t ) + alignment;
1606 descr.ptr_allocated = _malloc_src_loc( descr.size_allocated, _file_, _line_ );
1608 descr.ptr_allocated = malloc_src_loc( descr.size_allocated KMP_SRC_LOC_PARM );
1611 " malloc( %d ) returned %p\n",
1612 (
int) descr.size_allocated,
1615 if ( descr.ptr_allocated == NULL ) {
1616 KMP_FATAL( OutOfHeapMemory );
1619 addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
1621 ( addr_allocated +
sizeof( kmp_mem_descr_t ) + alignment )
1622 & ~ ( alignment - 1 );
1623 addr_descr = addr_aligned -
sizeof( kmp_mem_descr_t );
1625 descr.ptr_aligned = (
void *) addr_aligned;
1628 " ___kmp_allocate_align: " 1629 "ptr_allocated=%p, size_allocated=%d, " 1630 "ptr_aligned=%p, size_aligned=%d\n",
1631 descr.ptr_allocated,
1632 (
int) descr.size_allocated,
1634 (
int) descr.size_aligned
1637 KMP_DEBUG_ASSERT( addr_allocated <= addr_descr );
1638 KMP_DEBUG_ASSERT( addr_descr +
sizeof( kmp_mem_descr_t ) == addr_aligned );
1639 KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
1640 KMP_DEBUG_ASSERT( addr_aligned % alignment == 0 );
1643 memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
1646 memset( descr.ptr_aligned, 0x00, descr.size_aligned );
1650 * ( (kmp_mem_descr_t *) addr_descr ) = descr;
1654 KE_TRACE( 25, (
"<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned ) );
1655 return descr.ptr_aligned;
1667 ___kmp_allocate(
size_t size KMP_SRC_LOC_DECL )
1671 KE_TRACE( 25, (
"-> __kmp_allocate( %d ) called from %s:%d\n", (
int) size KMP_SRC_LOC_PARM ) );
1672 ptr = ___kmp_allocate_align( size, __kmp_align_alloc KMP_SRC_LOC_PARM );
1673 KE_TRACE( 25, (
"<- __kmp_allocate() returns %p\n", ptr ) );
1678 #if (BUILD_MEMORY==FIRST_TOUCH) 1680 __kmp_ft_page_allocate(
size_t size)
1685 int page_size = getpagesize();
1688 int page_size = PAGE_SIZE;
1691 adr = (
void *) __kmp_thread_malloc( __kmp_get_thread(),
1692 size + page_size + KMP_PTR_SKIP);
1694 KMP_FATAL( OutOfHeapMemory );
1697 if ( ( (kmp_uintptr_t) adr & (page_size - 1)) == 0)
1702 aadr = (
void *) ( ( (kmp_uintptr_t) adr + page_size) & ~(page_size - 1) );
1705 *((
void**)aadr) = adr;
1708 return (
void*)((
char*)aadr + KMP_PTR_SKIP);
1719 ___kmp_page_allocate(
size_t size KMP_SRC_LOC_DECL )
1721 int page_size = 8 * 1024;
1725 "-> __kmp_page_allocate( %d ) called from %s:%d\n",
1729 ptr = ___kmp_allocate_align( size, page_size KMP_SRC_LOC_PARM );
1730 KE_TRACE( 25, (
"<- __kmp_page_allocate( %d ) returns %p\n", (
int) size, ptr ) );
1739 ___kmp_free(
void * ptr KMP_SRC_LOC_DECL )
1742 kmp_mem_descr_t descr;
1743 kmp_uintptr_t addr_allocated;
1744 kmp_uintptr_t addr_aligned;
1746 KE_TRACE( 25, (
"-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM ) );
1747 KMP_ASSERT( ptr != NULL );
1749 descr = * ( kmp_mem_descr_t *) ( (kmp_uintptr_t) ptr -
sizeof( kmp_mem_descr_t ) );
1751 KE_TRACE( 26, (
" __kmp_free: " 1752 "ptr_allocated=%p, size_allocated=%d, " 1753 "ptr_aligned=%p, size_aligned=%d\n",
1754 descr.ptr_allocated, (
int) descr.size_allocated,
1755 descr.ptr_aligned, (
int) descr.size_aligned ));
1757 addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
1758 addr_aligned = (kmp_uintptr_t) descr.ptr_aligned;
1760 KMP_DEBUG_ASSERT( addr_aligned % CACHE_LINE == 0 );
1761 KMP_DEBUG_ASSERT( descr.ptr_aligned == ptr );
1762 KMP_DEBUG_ASSERT( addr_allocated +
sizeof( kmp_mem_descr_t ) <= addr_aligned );
1763 KMP_DEBUG_ASSERT( descr.size_aligned < descr.size_allocated );
1764 KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
1767 memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
1772 KE_TRACE( 10, (
" free( %p )\n", descr.ptr_allocated ) );
1774 _free_src_loc( descr.ptr_allocated, _file_, _line_ );
1776 free_src_loc( descr.ptr_allocated KMP_SRC_LOC_PARM );
1782 KE_TRACE( 25, (
"<- __kmp_free() returns\n" ) );
1789 #if USE_FAST_MEMORY == 3 1795 #define KMP_FREE_LIST_LIMIT 16 1798 #define DCACHE_LINE 128 1801 ___kmp_fast_allocate( kmp_info_t *this_thr,
size_t size KMP_SRC_LOC_DECL )
1809 kmp_mem_descr_t * descr;
1811 KE_TRACE( 25, (
"-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n",
1812 __kmp_gtid_from_thread(this_thr), (
int) size KMP_SRC_LOC_PARM ) );
1814 num_lines = ( size + DCACHE_LINE - 1 ) / DCACHE_LINE;
1815 idx = num_lines - 1;
1816 KMP_DEBUG_ASSERT( idx >= 0 );
1820 }
else if ( ( idx >>= 2 ) == 0 ) {
1823 }
else if ( ( idx >>= 2 ) == 0 ) {
1826 }
else if ( ( idx >>= 2 ) == 0 ) {
1833 ptr = this_thr->th.th_free_lists[index].th_free_list_self;
1834 if ( ptr != NULL ) {
1836 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1837 KMP_DEBUG_ASSERT( this_thr ==
1838 ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t) ))->ptr_aligned );
1841 ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
1842 if ( ptr != NULL ) {
1845 while ( ! KMP_COMPARE_AND_STORE_PTR(
1846 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, NULL ) )
1849 ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
1852 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1853 KMP_DEBUG_ASSERT( this_thr ==
1854 ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t) ))->ptr_aligned );
1860 size = num_lines * DCACHE_LINE;
1862 alloc_size = size +
sizeof( kmp_mem_descr_t ) + DCACHE_LINE;
1863 KE_TRACE( 25, (
"__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with alloc_size %d\n",
1864 __kmp_gtid_from_thread( this_thr ), alloc_size ) );
1865 alloc_ptr = bget( this_thr, (bufsize) alloc_size );
1868 ptr = (
void *)(( ((kmp_uintptr_t)alloc_ptr) +
sizeof(kmp_mem_descr_t) + DCACHE_LINE ) & ~( DCACHE_LINE - 1 ));
1869 descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t) );
1871 descr->ptr_allocated = alloc_ptr;
1873 descr->ptr_aligned = (
void *)this_thr;
1876 descr->size_aligned = size;
1879 KE_TRACE( 25, (
"<- __kmp_fast_allocate( T#%d ) returns %p\n",
1880 __kmp_gtid_from_thread( this_thr ), ptr ) );
1887 ___kmp_fast_free( kmp_info_t *this_thr,
void * ptr KMP_SRC_LOC_DECL )
1889 kmp_mem_descr_t * descr;
1890 kmp_info_t * alloc_thr;
1895 KE_TRACE( 25, (
"-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n",
1896 __kmp_gtid_from_thread(this_thr), ptr KMP_SRC_LOC_PARM ) );
1897 KMP_ASSERT( ptr != NULL );
1899 descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t) );
1901 KE_TRACE(26, (
" __kmp_fast_free: size_aligned=%d\n",
1902 (
int) descr->size_aligned ) );
1904 size = descr->size_aligned;
1906 idx = DCACHE_LINE * 2;
1907 if ( idx == size ) {
1909 }
else if ( ( idx <<= 1 ) == size ) {
1911 }
else if ( ( idx <<= 2 ) == size ) {
1913 }
else if ( ( idx <<= 2 ) == size ) {
1916 KMP_DEBUG_ASSERT( size > DCACHE_LINE * 64 );
1920 alloc_thr = (kmp_info_t *)descr->ptr_aligned;
1921 if ( alloc_thr == this_thr ) {
1923 *((
void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self;
1924 this_thr->th.th_free_lists[index].th_free_list_self = ptr;
1926 void * head = this_thr->th.th_free_lists[index].th_free_list_other;
1927 if ( head == NULL ) {
1929 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1930 *((
void **)ptr) = NULL;
1931 descr->size_allocated = (size_t)1;
1934 kmp_mem_descr_t * dsc = (kmp_mem_descr_t *)( (
char*)head -
sizeof(kmp_mem_descr_t) );
1935 kmp_info_t * q_th = (kmp_info_t *)(dsc->ptr_aligned);
1936 size_t q_sz = dsc->size_allocated + 1;
1937 if ( q_th == alloc_thr && q_sz <= KMP_FREE_LIST_LIMIT ) {
1939 *((
void **)ptr) = head;
1940 descr->size_allocated = q_sz;
1941 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1948 void * next = *((
void **)head);
1949 while ( next != NULL ) {
1952 ((kmp_mem_descr_t*)((
char*)next -
sizeof(kmp_mem_descr_t)))->size_allocated + 1 ==
1953 ((kmp_mem_descr_t*)((
char*)tail -
sizeof(kmp_mem_descr_t)))->size_allocated );
1955 next = *((
void **)next);
1957 KMP_DEBUG_ASSERT( q_th != NULL );
1959 old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
1962 *((
void **)tail) = old_ptr;
1964 while ( ! KMP_COMPARE_AND_STORE_PTR(
1965 &q_th->th.th_free_lists[index].th_free_list_sync,
1970 old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
1971 *((
void **)tail) = old_ptr;
1975 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1976 *((
void **)ptr) = NULL;
1977 descr->size_allocated = (size_t)1;
1984 KE_TRACE(25, (
"__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n",
1985 __kmp_gtid_from_thread( this_thr), size ) );
1986 __kmp_bget_dequeue( this_thr );
1987 brel( this_thr, descr->ptr_allocated );
1990 KE_TRACE( 25, (
"<- __kmp_fast_free() returns\n" ) );
1998 __kmp_initialize_fast_memory( kmp_info_t *this_thr )
2000 KE_TRACE(10, (
"__kmp_initialize_fast_memory: Called from th %p\n", this_thr ) );
2002 memset ( this_thr->th.th_free_lists, 0, NUM_LISTS *
sizeof( kmp_free_list_t ) );
2008 __kmp_free_fast_memory( kmp_info_t *th )
2012 thr_data_t * thr = get_thr_data( th );
2015 KE_TRACE(5, (
"__kmp_free_fast_memory: Called T#%d\n",
2016 __kmp_gtid_from_thread( th ) ) );
2018 __kmp_bget_dequeue( th );
2021 for ( bin = 0; bin < MAX_BGET_BINS; ++bin ) {
2022 bfhead_t * b = thr->freelist[ bin ].ql.flink;
2023 while ( b != &thr->freelist[ bin ] ) {
2024 if ( (kmp_uintptr_t)b->bh.bb.bthr & 1 ) {
2031 while ( lst != NULL ) {
2033 KE_TRACE(10, (
"__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n",
2034 lst, next, th, __kmp_gtid_from_thread( th ) ) );
2035 (*thr->relfcn)(lst);
2044 KE_TRACE(5, (
"__kmp_free_fast_memory: Freed T#%d\n",
2045 __kmp_gtid_from_thread( th ) ) );
2048 #endif // USE_FAST_MEMORY