24 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 26 # include <sys/syscall.h> 44 __kmp_validate_locks(
void )
50 x = ~((kmp_uint32) 0) - 2;
53 for (i = 0; i < 8; ++i, ++x, ++y) {
54 kmp_uint32 z = (x - y);
58 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
76 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
78 return DYNA_LOCK_STRIP(TCR_4( lck->lk.poll )) - 1;
82 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
84 return lck->lk.depth_locked != -1;
87 __forceinline
static void 88 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
92 #ifdef USE_LOCK_PROFILE 93 kmp_uint32 curr = TCR_4( lck->lk.poll );
94 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
95 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
99 if ( ( lck->lk.poll == DYNA_LOCK_FREE(tas) )
100 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) {
101 KMP_FSYNC_ACQUIRED(lck);
106 KMP_FSYNC_PREPARE( lck );
107 KMP_INIT_YIELD( spins );
108 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
113 KMP_YIELD_SPIN( spins );
116 while ( ( lck->lk.poll != DYNA_LOCK_FREE(tas) ) ||
117 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) ) {
121 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
126 KMP_YIELD_SPIN( spins );
129 KMP_FSYNC_ACQUIRED( lck );
133 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
135 __kmp_acquire_tas_lock_timed_template( lck, gtid );
139 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
141 char const *
const func =
"omp_set_lock";
142 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
143 && __kmp_is_tas_lock_nestable( lck ) ) {
144 KMP_FATAL( LockNestableUsedAsSimple, func );
146 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
147 KMP_FATAL( LockIsAlreadyOwned, func );
149 __kmp_acquire_tas_lock( lck, gtid );
153 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
155 if ( ( lck->lk.poll == DYNA_LOCK_FREE(tas) )
156 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) {
157 KMP_FSYNC_ACQUIRED( lck );
164 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
166 char const *
const func =
"omp_test_lock";
167 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
168 && __kmp_is_tas_lock_nestable( lck ) ) {
169 KMP_FATAL( LockNestableUsedAsSimple, func );
171 return __kmp_test_tas_lock( lck, gtid );
175 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
179 KMP_FSYNC_RELEASING(lck);
180 KMP_ST_REL32( &(lck->lk.poll), DYNA_LOCK_FREE(tas) );
183 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
185 return KMP_LOCK_RELEASED;
189 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
191 char const *
const func =
"omp_unset_lock";
193 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
194 && __kmp_is_tas_lock_nestable( lck ) ) {
195 KMP_FATAL( LockNestableUsedAsSimple, func );
197 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
198 KMP_FATAL( LockUnsettingFree, func );
200 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
201 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
202 KMP_FATAL( LockUnsettingSetByAnother, func );
204 return __kmp_release_tas_lock( lck, gtid );
208 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
210 TCW_4( lck->lk.poll, DYNA_LOCK_FREE(tas) );
214 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
216 __kmp_init_tas_lock( lck );
220 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
226 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
228 char const *
const func =
"omp_destroy_lock";
229 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
230 && __kmp_is_tas_lock_nestable( lck ) ) {
231 KMP_FATAL( LockNestableUsedAsSimple, func );
233 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
234 KMP_FATAL( LockStillOwned, func );
236 __kmp_destroy_tas_lock( lck );
245 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
247 KMP_DEBUG_ASSERT( gtid >= 0 );
249 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
250 lck->lk.depth_locked += 1;
253 __kmp_acquire_tas_lock_timed_template( lck, gtid );
254 lck->lk.depth_locked = 1;
259 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
261 char const *
const func =
"omp_set_nest_lock";
262 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
263 KMP_FATAL( LockSimpleUsedAsNestable, func );
265 __kmp_acquire_nested_tas_lock( lck, gtid );
269 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
273 KMP_DEBUG_ASSERT( gtid >= 0 );
275 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
276 retval = ++lck->lk.depth_locked;
278 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
283 retval = lck->lk.depth_locked = 1;
289 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
291 char const *
const func =
"omp_test_nest_lock";
292 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
293 KMP_FATAL( LockSimpleUsedAsNestable, func );
295 return __kmp_test_nested_tas_lock( lck, gtid );
299 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
301 KMP_DEBUG_ASSERT( gtid >= 0 );
304 if ( --(lck->lk.depth_locked) == 0 ) {
305 __kmp_release_tas_lock( lck, gtid );
306 return KMP_LOCK_RELEASED;
308 return KMP_LOCK_STILL_HELD;
312 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
314 char const *
const func =
"omp_unset_nest_lock";
316 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
317 KMP_FATAL( LockSimpleUsedAsNestable, func );
319 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
320 KMP_FATAL( LockUnsettingFree, func );
322 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
323 KMP_FATAL( LockUnsettingSetByAnother, func );
325 return __kmp_release_nested_tas_lock( lck, gtid );
329 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
331 __kmp_init_tas_lock( lck );
332 lck->lk.depth_locked = 0;
336 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
338 __kmp_init_nested_tas_lock( lck );
342 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
344 __kmp_destroy_tas_lock( lck );
345 lck->lk.depth_locked = 0;
349 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
351 char const *
const func =
"omp_destroy_nest_lock";
352 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
353 KMP_FATAL( LockSimpleUsedAsNestable, func );
355 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
356 KMP_FATAL( LockStillOwned, func );
358 __kmp_destroy_nested_tas_lock( lck );
362 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 373 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
375 return DYNA_LOCK_STRIP(( TCR_4( lck->lk.poll ) >> 1 )) - 1;
379 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
381 return lck->lk.depth_locked != -1;
384 __forceinline
static void 385 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
387 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
391 #ifdef USE_LOCK_PROFILE 392 kmp_uint32 curr = TCR_4( lck->lk.poll );
393 if ( ( curr != 0 ) && ( curr != gtid_code ) )
394 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
398 KMP_FSYNC_PREPARE( lck );
399 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
400 lck, lck->lk.poll, gtid ) );
404 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex),
405 DYNA_LOCK_BUSY(gtid_code, futex) ) ) != DYNA_LOCK_FREE(futex) ) {
407 kmp_int32 cond = DYNA_LOCK_STRIP(poll_val) & 1;
408 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
409 lck, gtid, poll_val, cond ) );
424 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ), poll_val, poll_val | DYNA_LOCK_BUSY(1, futex) ) ) {
425 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
426 lck, lck->lk.poll, gtid ) );
429 poll_val |= DYNA_LOCK_BUSY(1, futex);
431 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
432 lck, lck->lk.poll, gtid ) );
435 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
436 lck, gtid, poll_val ) );
439 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
440 poll_val, NULL, NULL, 0 ) ) != 0 ) {
441 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
442 lck, gtid, poll_val, rc, errno ) );
446 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
447 lck, gtid, poll_val ) );
457 KMP_FSYNC_ACQUIRED( lck );
458 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
459 lck, lck->lk.poll, gtid ) );
463 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
465 __kmp_acquire_futex_lock_timed_template( lck, gtid );
469 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
471 char const *
const func =
"omp_set_lock";
472 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
473 && __kmp_is_futex_lock_nestable( lck ) ) {
474 KMP_FATAL( LockNestableUsedAsSimple, func );
476 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
477 KMP_FATAL( LockIsAlreadyOwned, func );
479 __kmp_acquire_futex_lock( lck, gtid );
483 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
485 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex), DYNA_LOCK_BUSY(gtid+1, futex) << 1 ) ) {
486 KMP_FSYNC_ACQUIRED( lck );
493 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
495 char const *
const func =
"omp_test_lock";
496 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
497 && __kmp_is_futex_lock_nestable( lck ) ) {
498 KMP_FATAL( LockNestableUsedAsSimple, func );
500 return __kmp_test_futex_lock( lck, gtid );
504 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
508 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
509 lck, lck->lk.poll, gtid ) );
511 KMP_FSYNC_RELEASING(lck);
513 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex) );
515 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
516 lck, gtid, poll_val ) );
518 if ( DYNA_LOCK_STRIP(poll_val) & 1 ) {
519 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
521 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, DYNA_LOCK_BUSY(1, futex), NULL, NULL, 0 );
526 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
527 lck, lck->lk.poll, gtid ) );
529 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
531 return KMP_LOCK_RELEASED;
535 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
537 char const *
const func =
"omp_unset_lock";
539 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
540 && __kmp_is_futex_lock_nestable( lck ) ) {
541 KMP_FATAL( LockNestableUsedAsSimple, func );
543 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
544 KMP_FATAL( LockUnsettingFree, func );
546 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
547 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
548 KMP_FATAL( LockUnsettingSetByAnother, func );
550 return __kmp_release_futex_lock( lck, gtid );
554 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
556 TCW_4( lck->lk.poll, DYNA_LOCK_FREE(futex) );
560 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
562 __kmp_init_futex_lock( lck );
566 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
572 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
574 char const *
const func =
"omp_destroy_lock";
575 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
576 && __kmp_is_futex_lock_nestable( lck ) ) {
577 KMP_FATAL( LockNestableUsedAsSimple, func );
579 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
580 KMP_FATAL( LockStillOwned, func );
582 __kmp_destroy_futex_lock( lck );
591 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
593 KMP_DEBUG_ASSERT( gtid >= 0 );
595 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
596 lck->lk.depth_locked += 1;
599 __kmp_acquire_futex_lock_timed_template( lck, gtid );
600 lck->lk.depth_locked = 1;
605 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
607 char const *
const func =
"omp_set_nest_lock";
608 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
609 KMP_FATAL( LockSimpleUsedAsNestable, func );
611 __kmp_acquire_nested_futex_lock( lck, gtid );
615 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
619 KMP_DEBUG_ASSERT( gtid >= 0 );
621 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
622 retval = ++lck->lk.depth_locked;
624 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
629 retval = lck->lk.depth_locked = 1;
635 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
637 char const *
const func =
"omp_test_nest_lock";
638 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
639 KMP_FATAL( LockSimpleUsedAsNestable, func );
641 return __kmp_test_nested_futex_lock( lck, gtid );
645 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
647 KMP_DEBUG_ASSERT( gtid >= 0 );
650 if ( --(lck->lk.depth_locked) == 0 ) {
651 __kmp_release_futex_lock( lck, gtid );
652 return KMP_LOCK_RELEASED;
654 return KMP_LOCK_STILL_HELD;
658 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
660 char const *
const func =
"omp_unset_nest_lock";
662 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
663 KMP_FATAL( LockSimpleUsedAsNestable, func );
665 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
666 KMP_FATAL( LockUnsettingFree, func );
668 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
669 KMP_FATAL( LockUnsettingSetByAnother, func );
671 return __kmp_release_nested_futex_lock( lck, gtid );
675 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
677 __kmp_init_futex_lock( lck );
678 lck->lk.depth_locked = 0;
682 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
684 __kmp_init_nested_futex_lock( lck );
688 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
690 __kmp_destroy_futex_lock( lck );
691 lck->lk.depth_locked = 0;
695 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
697 char const *
const func =
"omp_destroy_nest_lock";
698 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
699 KMP_FATAL( LockSimpleUsedAsNestable, func );
701 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
702 KMP_FATAL( LockStillOwned, func );
704 __kmp_destroy_nested_futex_lock( lck );
707 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) 714 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
716 return TCR_4( lck->lk.owner_id ) - 1;
720 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
722 return lck->lk.depth_locked != -1;
726 __kmp_bakery_check(kmp_uint value, kmp_uint checker)
728 register kmp_uint32 pause;
730 if (value == checker) {
733 for (pause = checker - value; pause != 0; --pause);
737 __forceinline
static void 738 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
740 kmp_uint32 my_ticket;
743 my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
745 #ifdef USE_LOCK_PROFILE 746 if ( TCR_4( lck->lk.now_serving ) != my_ticket )
747 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
751 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
752 KMP_FSYNC_ACQUIRED(lck);
755 KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
756 KMP_FSYNC_ACQUIRED(lck);
760 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
762 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
766 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
768 char const *
const func =
"omp_set_lock";
769 if ( lck->lk.initialized != lck ) {
770 KMP_FATAL( LockIsUninitialized, func );
772 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
773 KMP_FATAL( LockNestableUsedAsSimple, func );
775 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
776 KMP_FATAL( LockIsAlreadyOwned, func );
779 __kmp_acquire_ticket_lock( lck, gtid );
781 lck->lk.owner_id = gtid + 1;
785 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
787 kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
788 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
789 kmp_uint32 next_ticket = my_ticket + 1;
790 if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
791 my_ticket, next_ticket ) ) {
792 KMP_FSYNC_ACQUIRED( lck );
800 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
802 char const *
const func =
"omp_test_lock";
803 if ( lck->lk.initialized != lck ) {
804 KMP_FATAL( LockIsUninitialized, func );
806 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
807 KMP_FATAL( LockNestableUsedAsSimple, func );
810 int retval = __kmp_test_ticket_lock( lck, gtid );
813 lck->lk.owner_id = gtid + 1;
819 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
825 KMP_FSYNC_RELEASING(lck);
826 distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
828 KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
833 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
834 return KMP_LOCK_RELEASED;
838 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
840 char const *
const func =
"omp_unset_lock";
842 if ( lck->lk.initialized != lck ) {
843 KMP_FATAL( LockIsUninitialized, func );
845 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
846 KMP_FATAL( LockNestableUsedAsSimple, func );
848 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
849 KMP_FATAL( LockUnsettingFree, func );
851 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
852 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
853 KMP_FATAL( LockUnsettingSetByAnother, func );
855 lck->lk.owner_id = 0;
856 return __kmp_release_ticket_lock( lck, gtid );
860 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
862 lck->lk.location = NULL;
863 TCW_4( lck->lk.next_ticket, 0 );
864 TCW_4( lck->lk.now_serving, 0 );
865 lck->lk.owner_id = 0;
866 lck->lk.depth_locked = -1;
867 lck->lk.initialized = (kmp_ticket_lock *)lck;
871 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
873 __kmp_init_ticket_lock( lck );
877 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
879 lck->lk.initialized = NULL;
880 lck->lk.location = NULL;
881 lck->lk.next_ticket = 0;
882 lck->lk.now_serving = 0;
883 lck->lk.owner_id = 0;
884 lck->lk.depth_locked = -1;
888 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
890 char const *
const func =
"omp_destroy_lock";
891 if ( lck->lk.initialized != lck ) {
892 KMP_FATAL( LockIsUninitialized, func );
894 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
895 KMP_FATAL( LockNestableUsedAsSimple, func );
897 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
898 KMP_FATAL( LockStillOwned, func );
900 __kmp_destroy_ticket_lock( lck );
909 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
911 KMP_DEBUG_ASSERT( gtid >= 0 );
913 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
914 lck->lk.depth_locked += 1;
917 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
919 lck->lk.depth_locked = 1;
921 lck->lk.owner_id = gtid + 1;
926 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
928 char const *
const func =
"omp_set_nest_lock";
929 if ( lck->lk.initialized != lck ) {
930 KMP_FATAL( LockIsUninitialized, func );
932 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
933 KMP_FATAL( LockSimpleUsedAsNestable, func );
935 __kmp_acquire_nested_ticket_lock( lck, gtid );
939 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
943 KMP_DEBUG_ASSERT( gtid >= 0 );
945 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
946 retval = ++lck->lk.depth_locked;
948 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
953 retval = lck->lk.depth_locked = 1;
955 lck->lk.owner_id = gtid + 1;
961 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
964 char const *
const func =
"omp_test_nest_lock";
965 if ( lck->lk.initialized != lck ) {
966 KMP_FATAL( LockIsUninitialized, func );
968 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
969 KMP_FATAL( LockSimpleUsedAsNestable, func );
971 return __kmp_test_nested_ticket_lock( lck, gtid );
975 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
977 KMP_DEBUG_ASSERT( gtid >= 0 );
980 if ( --(lck->lk.depth_locked) == 0 ) {
982 lck->lk.owner_id = 0;
983 __kmp_release_ticket_lock( lck, gtid );
984 return KMP_LOCK_RELEASED;
986 return KMP_LOCK_STILL_HELD;
990 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
992 char const *
const func =
"omp_unset_nest_lock";
994 if ( lck->lk.initialized != lck ) {
995 KMP_FATAL( LockIsUninitialized, func );
997 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
998 KMP_FATAL( LockSimpleUsedAsNestable, func );
1000 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1001 KMP_FATAL( LockUnsettingFree, func );
1003 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1004 KMP_FATAL( LockUnsettingSetByAnother, func );
1006 return __kmp_release_nested_ticket_lock( lck, gtid );
1010 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1012 __kmp_init_ticket_lock( lck );
1013 lck->lk.depth_locked = 0;
1017 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1019 __kmp_init_nested_ticket_lock( lck );
1023 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1025 __kmp_destroy_ticket_lock( lck );
1026 lck->lk.depth_locked = 0;
1030 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1032 char const *
const func =
"omp_destroy_nest_lock";
1033 if ( lck->lk.initialized != lck ) {
1034 KMP_FATAL( LockIsUninitialized, func );
1036 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1037 KMP_FATAL( LockSimpleUsedAsNestable, func );
1039 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1040 KMP_FATAL( LockStillOwned, func );
1042 __kmp_destroy_nested_ticket_lock( lck );
1051 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1053 return lck == lck->lk.initialized;
1057 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1059 return lck->lk.location;
1063 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1065 lck->lk.location = loc;
1068 static kmp_lock_flags_t
1069 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1071 return lck->lk.flags;
1075 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1077 lck->lk.flags = flags;
1136 #ifdef DEBUG_QUEUING_LOCKS 1139 #define TRACE_BUF_ELE 1024 1140 static char traces[TRACE_BUF_ELE][128] = { 0 }
1142 #define TRACE_LOCK(X,Y) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y ); 1143 #define TRACE_LOCK_T(X,Y,Z) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X,Y,Z ); 1144 #define TRACE_LOCK_HT(X,Y,Z,Q) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, Z, Q ); 1147 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1148 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1152 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1154 i = tc % TRACE_BUF_ELE;
1155 __kmp_printf_no_lock(
"%s\n", traces[i] );
1156 i = (i+1) % TRACE_BUF_ELE;
1157 while ( i != (tc % TRACE_BUF_ELE) ) {
1158 __kmp_printf_no_lock(
"%s", traces[i] );
1159 i = (i+1) % TRACE_BUF_ELE;
1161 __kmp_printf_no_lock(
"\n" );
1163 __kmp_printf_no_lock(
1164 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1165 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1168 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1170 if ( lck->lk.head_id >= 1 ) {
1171 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1173 __kmp_printf_no_lock(
"-> %d ", t );
1174 t = __kmp_threads[t-1]->th.th_next_waiting;
1177 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1178 __kmp_printf_no_lock(
"\n\n" );
1184 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1186 return TCR_4( lck->lk.owner_id ) - 1;
1190 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1192 return lck->lk.depth_locked != -1;
1196 template <
bool takeTime>
1199 __forceinline
static void 1200 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1203 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1204 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1205 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1206 volatile kmp_uint32 *spin_here_p;
1207 kmp_int32 need_mf = 1;
1210 ompt_state_t prev_state = ompt_state_undefined;
1213 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1215 KMP_FSYNC_PREPARE( lck );
1216 KMP_DEBUG_ASSERT( this_thr != NULL );
1217 spin_here_p = & this_thr->th.th_spin_here;
1219 #ifdef DEBUG_QUEUING_LOCKS 1220 TRACE_LOCK( gtid+1,
"acq ent" );
1222 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1223 if ( this_thr->th.th_next_waiting != 0 )
1224 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1226 KMP_DEBUG_ASSERT( !*spin_here_p );
1227 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1237 *spin_here_p = TRUE;
1250 #ifdef DEBUG_QUEUING_LOCKS 1252 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1261 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1262 KMP_PACK_64( -1, 0 ),
1263 KMP_PACK_64( gtid+1, gtid+1 ) );
1264 #ifdef DEBUG_QUEUING_LOCKS 1265 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1273 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1275 #ifdef DEBUG_QUEUING_LOCKS 1276 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1285 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1287 #ifdef DEBUG_QUEUING_LOCKS 1288 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1296 kmp_int32 grabbed_lock;
1298 #ifdef DEBUG_QUEUING_LOCKS 1300 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1305 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1307 if ( grabbed_lock ) {
1309 *spin_here_p = FALSE;
1311 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1313 #ifdef DEBUG_QUEUING_LOCKS 1314 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1318 if ((ompt_status & ompt_status_track) &&
1319 prev_state != ompt_state_undefined) {
1321 this_thr->th.ompt_thread_info.state = prev_state;
1322 this_thr->th.ompt_thread_info.wait_id = 0;
1326 KMP_FSYNC_ACQUIRED( lck );
1335 if ((ompt_status & ompt_status_track) &&
1336 prev_state == ompt_state_undefined) {
1338 prev_state = this_thr->th.ompt_thread_info.state;
1339 this_thr->th.ompt_thread_info.wait_id = (uint64_t) lck;
1340 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1346 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1347 KMP_ASSERT( tail_thr != NULL );
1348 tail_thr->th.th_next_waiting = gtid+1;
1351 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1358 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1360 #ifdef DEBUG_QUEUING_LOCKS 1361 TRACE_LOCK( gtid+1,
"acq spin" );
1363 if ( this_thr->th.th_next_waiting != 0 )
1364 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1366 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1367 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1370 #ifdef DEBUG_QUEUING_LOCKS 1371 TRACE_LOCK( gtid+1,
"acq exit 2" );
1376 this_thr->th.ompt_thread_info.state = prev_state;
1377 this_thr->th.ompt_thread_info.wait_id = 0;
1387 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1389 #ifdef DEBUG_QUEUING_LOCKS 1390 TRACE_LOCK( gtid+1,
"acq retry" );
1394 KMP_ASSERT2( 0,
"should not get here" );
1398 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1400 KMP_DEBUG_ASSERT( gtid >= 0 );
1402 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1406 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1409 char const *
const func =
"omp_set_lock";
1410 if ( lck->lk.initialized != lck ) {
1411 KMP_FATAL( LockIsUninitialized, func );
1413 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1414 KMP_FATAL( LockNestableUsedAsSimple, func );
1416 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1417 KMP_FATAL( LockIsAlreadyOwned, func );
1420 __kmp_acquire_queuing_lock( lck, gtid );
1422 lck->lk.owner_id = gtid + 1;
1426 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1428 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1431 kmp_info_t *this_thr;
1434 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1435 KMP_DEBUG_ASSERT( gtid >= 0 );
1437 this_thr = __kmp_thread_from_gtid( gtid );
1438 KMP_DEBUG_ASSERT( this_thr != NULL );
1439 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1448 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1449 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1450 KMP_FSYNC_ACQUIRED(lck);
1455 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1460 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1462 char const *
const func =
"omp_test_lock";
1463 if ( lck->lk.initialized != lck ) {
1464 KMP_FATAL( LockIsUninitialized, func );
1466 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1467 KMP_FATAL( LockNestableUsedAsSimple, func );
1470 int retval = __kmp_test_queuing_lock( lck, gtid );
1473 lck->lk.owner_id = gtid + 1;
1479 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1481 register kmp_info_t *this_thr;
1482 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1483 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1485 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1486 KMP_DEBUG_ASSERT( gtid >= 0 );
1487 this_thr = __kmp_thread_from_gtid( gtid );
1488 KMP_DEBUG_ASSERT( this_thr != NULL );
1489 #ifdef DEBUG_QUEUING_LOCKS 1490 TRACE_LOCK( gtid+1,
"rel ent" );
1492 if ( this_thr->th.th_spin_here )
1493 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1494 if ( this_thr->th.th_next_waiting != 0 )
1495 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1497 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1498 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1500 KMP_FSYNC_RELEASING(lck);
1509 #ifdef DEBUG_QUEUING_LOCKS 1511 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1512 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1514 KMP_DEBUG_ASSERT( head != 0 );
1519 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1520 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1522 #ifdef DEBUG_QUEUING_LOCKS 1523 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1530 return KMP_LOCK_RELEASED;
1538 if ( head == tail ) {
1540 #ifdef DEBUG_QUEUING_LOCKS 1541 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1543 KMP_DEBUG_ASSERT( head > 0 );
1546 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1547 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1548 #ifdef DEBUG_QUEUING_LOCKS 1549 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1554 volatile kmp_int32 *waiting_id_p;
1555 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1556 KMP_DEBUG_ASSERT( head_thr != NULL );
1557 waiting_id_p = & head_thr->th.th_next_waiting;
1560 #ifdef DEBUG_QUEUING_LOCKS 1561 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1563 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1569 *head_id_p = (kmp_int32) KMP_WAIT_YIELD((
volatile kmp_uint*) waiting_id_p, 0, KMP_NEQ, NULL);
1570 #ifdef DEBUG_QUEUING_LOCKS 1571 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1578 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1579 KMP_DEBUG_ASSERT( head_thr != NULL );
1582 #ifdef DEBUG_QUEUING_LOCKS 1583 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1585 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1590 head_thr->th.th_next_waiting = 0;
1591 #ifdef DEBUG_QUEUING_LOCKS 1592 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1597 head_thr->th.th_spin_here = FALSE;
1599 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1601 #ifdef DEBUG_QUEUING_LOCKS 1602 TRACE_LOCK( gtid+1,
"rel exit 2" );
1604 return KMP_LOCK_RELEASED;
1608 #ifdef DEBUG_QUEUING_LOCKS 1609 TRACE_LOCK( gtid+1,
"rel retry" );
1613 KMP_ASSERT2( 0,
"should not get here" );
1614 return KMP_LOCK_RELEASED;
1618 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1621 char const *
const func =
"omp_unset_lock";
1623 if ( lck->lk.initialized != lck ) {
1624 KMP_FATAL( LockIsUninitialized, func );
1626 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1627 KMP_FATAL( LockNestableUsedAsSimple, func );
1629 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1630 KMP_FATAL( LockUnsettingFree, func );
1632 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1633 KMP_FATAL( LockUnsettingSetByAnother, func );
1635 lck->lk.owner_id = 0;
1636 return __kmp_release_queuing_lock( lck, gtid );
1640 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1642 lck->lk.location = NULL;
1643 lck->lk.head_id = 0;
1644 lck->lk.tail_id = 0;
1645 lck->lk.next_ticket = 0;
1646 lck->lk.now_serving = 0;
1647 lck->lk.owner_id = 0;
1648 lck->lk.depth_locked = -1;
1649 lck->lk.initialized = lck;
1651 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1655 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1657 __kmp_init_queuing_lock( lck );
1661 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1663 lck->lk.initialized = NULL;
1664 lck->lk.location = NULL;
1665 lck->lk.head_id = 0;
1666 lck->lk.tail_id = 0;
1667 lck->lk.next_ticket = 0;
1668 lck->lk.now_serving = 0;
1669 lck->lk.owner_id = 0;
1670 lck->lk.depth_locked = -1;
1674 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1676 char const *
const func =
"omp_destroy_lock";
1677 if ( lck->lk.initialized != lck ) {
1678 KMP_FATAL( LockIsUninitialized, func );
1680 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1681 KMP_FATAL( LockNestableUsedAsSimple, func );
1683 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1684 KMP_FATAL( LockStillOwned, func );
1686 __kmp_destroy_queuing_lock( lck );
1695 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1697 KMP_DEBUG_ASSERT( gtid >= 0 );
1699 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1700 lck->lk.depth_locked += 1;
1703 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1705 lck->lk.depth_locked = 1;
1707 lck->lk.owner_id = gtid + 1;
1712 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1714 char const *
const func =
"omp_set_nest_lock";
1715 if ( lck->lk.initialized != lck ) {
1716 KMP_FATAL( LockIsUninitialized, func );
1718 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1719 KMP_FATAL( LockSimpleUsedAsNestable, func );
1721 __kmp_acquire_nested_queuing_lock( lck, gtid );
1725 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1729 KMP_DEBUG_ASSERT( gtid >= 0 );
1731 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1732 retval = ++lck->lk.depth_locked;
1734 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1739 retval = lck->lk.depth_locked = 1;
1741 lck->lk.owner_id = gtid + 1;
1747 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1750 char const *
const func =
"omp_test_nest_lock";
1751 if ( lck->lk.initialized != lck ) {
1752 KMP_FATAL( LockIsUninitialized, func );
1754 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1755 KMP_FATAL( LockSimpleUsedAsNestable, func );
1757 return __kmp_test_nested_queuing_lock( lck, gtid );
1761 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1763 KMP_DEBUG_ASSERT( gtid >= 0 );
1766 if ( --(lck->lk.depth_locked) == 0 ) {
1768 lck->lk.owner_id = 0;
1769 __kmp_release_queuing_lock( lck, gtid );
1770 return KMP_LOCK_RELEASED;
1772 return KMP_LOCK_STILL_HELD;
1776 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1778 char const *
const func =
"omp_unset_nest_lock";
1780 if ( lck->lk.initialized != lck ) {
1781 KMP_FATAL( LockIsUninitialized, func );
1783 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1784 KMP_FATAL( LockSimpleUsedAsNestable, func );
1786 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1787 KMP_FATAL( LockUnsettingFree, func );
1789 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1790 KMP_FATAL( LockUnsettingSetByAnother, func );
1792 return __kmp_release_nested_queuing_lock( lck, gtid );
1796 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1798 __kmp_init_queuing_lock( lck );
1799 lck->lk.depth_locked = 0;
1803 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1805 __kmp_init_nested_queuing_lock( lck );
1809 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1811 __kmp_destroy_queuing_lock( lck );
1812 lck->lk.depth_locked = 0;
1816 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1818 char const *
const func =
"omp_destroy_nest_lock";
1819 if ( lck->lk.initialized != lck ) {
1820 KMP_FATAL( LockIsUninitialized, func );
1822 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1823 KMP_FATAL( LockSimpleUsedAsNestable, func );
1825 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1826 KMP_FATAL( LockStillOwned, func );
1828 __kmp_destroy_nested_queuing_lock( lck );
1837 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1839 return lck == lck->lk.initialized;
1843 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1845 return lck->lk.location;
1849 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1851 lck->lk.location = loc;
1854 static kmp_lock_flags_t
1855 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1857 return lck->lk.flags;
1861 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1863 lck->lk.flags = flags;
1866 #if KMP_USE_ADAPTIVE_LOCKS 1876 #define _XBEGIN_STARTED (~0u) 1877 #define _XABORT_EXPLICIT (1 << 0) 1878 #define _XABORT_RETRY (1 << 1) 1879 #define _XABORT_CONFLICT (1 << 2) 1880 #define _XABORT_CAPACITY (1 << 3) 1881 #define _XABORT_DEBUG (1 << 4) 1882 #define _XABORT_NESTED (1 << 5) 1883 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1886 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1888 #define STRINGIZE_INTERNAL(arg) #arg 1889 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1897 static __inline
int _xbegin()
1926 #endif // KMP_ARCH_X86_64 1936 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n" 1939 "1: movl %%eax,%0\n" 1941 :
"+r"(res)::
"memory",
"%eax");
1942 #endif // KMP_OS_WINDOWS 1949 static __inline
void _xend()
1958 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1968 #define _xabort(ARG) \ 1973 #define _xabort(ARG) \ 1974 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory"); 1980 #if KMP_DEBUG_ADAPTIVE_LOCKS 1985 static kmp_adaptive_lock_statistics_t destroyedStats;
1988 static kmp_adaptive_lock_info_t liveLocks;
1991 static kmp_bootstrap_lock_t chain_lock;
1995 __kmp_init_speculative_stats()
1997 kmp_adaptive_lock_info_t *lck = &liveLocks;
1999 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
2000 lck->stats.next = lck;
2001 lck->stats.prev = lck;
2003 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2004 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2006 __kmp_init_bootstrap_lock( &chain_lock );
2012 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
2014 __kmp_acquire_bootstrap_lock( &chain_lock );
2016 lck->stats.next = liveLocks.stats.next;
2017 lck->stats.prev = &liveLocks;
2019 liveLocks.stats.next = lck;
2020 lck->stats.next->stats.prev = lck;
2022 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2023 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2025 __kmp_release_bootstrap_lock( &chain_lock );
2029 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2031 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2032 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2034 kmp_adaptive_lock_info_t * n = lck->stats.next;
2035 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2042 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2044 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2045 __kmp_remember_lock( lck );
2049 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2051 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2053 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2054 t->successfulSpeculations += s->successfulSpeculations;
2055 t->hardFailedSpeculations += s->hardFailedSpeculations;
2056 t->softFailedSpeculations += s->softFailedSpeculations;
2057 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2058 t->lemmingYields += s->lemmingYields;
2062 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2064 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2066 __kmp_acquire_bootstrap_lock( &chain_lock );
2068 __kmp_add_stats( &destroyedStats, lck );
2069 __kmp_forget_lock( lck );
2071 __kmp_release_bootstrap_lock( &chain_lock );
2075 percent (kmp_uint32 count, kmp_uint32 total)
2077 return (total == 0) ? 0.0: (100.0 * count)/total;
2081 FILE * __kmp_open_stats_file()
2083 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2086 size_t buffLen = KMP_STRLEN( __kmp_speculative_statsfile ) + 20;
2087 char buffer[buffLen];
2088 KMP_SNPRINTF (&buffer[0], buffLen, __kmp_speculative_statsfile,
2089 (kmp_int32)getpid());
2090 FILE * result = fopen(&buffer[0],
"w");
2093 return result ? result : stdout;
2097 __kmp_print_speculative_stats()
2099 if (__kmp_user_lock_kind != lk_adaptive)
2102 FILE * statsFile = __kmp_open_stats_file();
2104 kmp_adaptive_lock_statistics_t total = destroyedStats;
2105 kmp_adaptive_lock_info_t *lck;
2107 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2108 __kmp_add_stats( &total, lck );
2110 kmp_adaptive_lock_statistics_t *t = &total;
2111 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2112 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2113 t->softFailedSpeculations;
2115 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2116 fprintf ( statsFile,
" Lock parameters: \n" 2117 " max_soft_retries : %10d\n" 2118 " max_badness : %10d\n",
2119 __kmp_adaptive_backoff_params.max_soft_retries,
2120 __kmp_adaptive_backoff_params.max_badness);
2121 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2122 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2123 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2124 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2125 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2126 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2127 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2129 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2130 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2131 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2132 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2133 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2134 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2135 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2137 if (statsFile != stdout)
2138 fclose( statsFile );
2141 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ ) 2143 # define KMP_INC_STAT(lck,stat) 2145 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2148 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2152 bool res = lck->lk.head_id == 0;
2156 #if KMP_COMPILER_ICC 2159 __sync_synchronize();
2166 static __inline
void 2167 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2170 lck->lk.adaptive.badness = 0;
2171 KMP_INC_STAT(lck,successfulSpeculations);
2175 static __inline
void 2176 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2178 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2179 if ( newBadness > lck->lk.adaptive.max_badness) {
2182 lck->lk.adaptive.badness = newBadness;
2188 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2190 kmp_uint32 badness = lck->lk.adaptive.badness;
2191 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2192 int res = (attempts & badness) == 0;
2200 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2202 int retries = lck->lk.adaptive.max_soft_retries;
2210 kmp_uint32 status = _xbegin();
2215 if (status == _XBEGIN_STARTED )
2221 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2227 KMP_ASSERT2( 0, "should not get here" );
2232 if ( status & SOFT_ABORT_MASK)
2234 KMP_INC_STAT(lck,softFailedSpeculations);
2239 KMP_INC_STAT(lck,hardFailedSpeculations);
2244 }
while( retries-- );
2248 __kmp_step_badness( lck );
2256 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2259 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2264 lck->lk.adaptive.acquire_attempts++;
2267 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2269 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2279 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2281 char const *
const func =
"omp_test_lock";
2282 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2283 KMP_FATAL( LockIsUninitialized, func );
2286 int retval = __kmp_test_adaptive_lock( lck, gtid );
2289 lck->lk.qlk.owner_id = gtid + 1;
2309 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2311 if ( __kmp_should_speculate( lck, gtid ) )
2313 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2315 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2327 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2329 KMP_INC_STAT(lck,lemmingYields);
2333 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2340 lck->lk.adaptive.acquire_attempts++;
2342 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2344 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2348 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2350 char const *
const func =
"omp_set_lock";
2351 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2352 KMP_FATAL( LockIsUninitialized, func );
2354 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2355 KMP_FATAL( LockIsAlreadyOwned, func );
2358 __kmp_acquire_adaptive_lock( lck, gtid );
2360 lck->lk.qlk.owner_id = gtid + 1;
2364 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2366 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2371 __kmp_update_badness_after_success( lck );
2376 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2378 return KMP_LOCK_RELEASED;
2382 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2384 char const *
const func =
"omp_unset_lock";
2386 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2387 KMP_FATAL( LockIsUninitialized, func );
2389 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2390 KMP_FATAL( LockUnsettingFree, func );
2392 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2393 KMP_FATAL( LockUnsettingSetByAnother, func );
2395 lck->lk.qlk.owner_id = 0;
2396 __kmp_release_adaptive_lock( lck, gtid );
2397 return KMP_LOCK_RELEASED;
2401 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2403 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2404 lck->lk.adaptive.badness = 0;
2405 lck->lk.adaptive.acquire_attempts = 0;
2406 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2407 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2408 #if KMP_DEBUG_ADAPTIVE_LOCKS 2409 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2411 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2415 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2417 __kmp_init_adaptive_lock( lck );
2421 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2423 #if KMP_DEBUG_ADAPTIVE_LOCKS 2424 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2426 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2431 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2433 char const *
const func =
"omp_destroy_lock";
2434 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2435 KMP_FATAL( LockIsUninitialized, func );
2437 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2438 KMP_FATAL( LockStillOwned, func );
2440 __kmp_destroy_adaptive_lock( lck );
2444 #endif // KMP_USE_ADAPTIVE_LOCKS 2452 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2454 return TCR_4( lck->lk.owner_id ) - 1;
2458 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2460 return lck->lk.depth_locked != -1;
2463 __forceinline
static void 2464 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2466 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2467 kmp_uint64 mask = TCR_8(lck->lk.mask);
2468 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2469 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2470 TCR_PTR(lck->lk.polls);
2472 #ifdef USE_LOCK_PROFILE 2473 if (TCR_8(polls[ticket & mask].poll) != ticket)
2474 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2490 KMP_FSYNC_PREPARE(lck);
2491 KMP_INIT_YIELD(spins);
2492 while (TCR_8(polls[ticket & mask]).poll < ticket) {
2497 KMP_YIELD(TCR_4(__kmp_nth)
2498 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2499 KMP_YIELD_SPIN(spins);
2510 mask = TCR_8(lck->lk.mask);
2511 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2512 TCR_PTR(lck->lk.polls);
2518 KMP_FSYNC_ACQUIRED(lck);
2519 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2521 lck->lk.now_serving = ticket;
2530 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2531 __kmp_free((
void *)lck->lk.old_polls);
2532 lck->lk.old_polls = NULL;
2533 lck->lk.cleanup_ticket = 0;
2541 if (lck->lk.old_polls == NULL) {
2542 bool reconfigure =
false;
2543 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2544 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2546 if (TCR_4(__kmp_nth)
2547 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2552 if (num_polls > 1) {
2554 num_polls = TCR_4(lck->lk.num_polls);
2557 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2558 __kmp_allocate(num_polls *
sizeof(*polls));
2559 polls[0].poll = ticket;
2568 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2569 if (num_waiting > num_polls) {
2570 kmp_uint32 old_num_polls = num_polls;
2573 mask = (mask << 1) | 1;
2575 }
while (num_polls <= num_waiting);
2583 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2584 __kmp_allocate(num_polls *
sizeof(*polls));
2586 for (i = 0; i < old_num_polls; i++) {
2587 polls[i].poll = old_polls[i].poll;
2604 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2605 ticket, lck, num_polls));
2607 lck->lk.old_polls = old_polls;
2608 lck->lk.polls = polls;
2612 lck->lk.num_polls = num_polls;
2613 lck->lk.mask = mask;
2623 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2629 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2631 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2635 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2637 char const *
const func =
"omp_set_lock";
2638 if ( lck->lk.initialized != lck ) {
2639 KMP_FATAL( LockIsUninitialized, func );
2641 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2642 KMP_FATAL( LockNestableUsedAsSimple, func );
2644 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2645 KMP_FATAL( LockIsAlreadyOwned, func );
2648 __kmp_acquire_drdpa_lock( lck, gtid );
2650 lck->lk.owner_id = gtid + 1;
2654 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2660 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2661 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2662 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2663 TCR_PTR(lck->lk.polls);
2664 kmp_uint64 mask = TCR_8(lck->lk.mask);
2665 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2666 kmp_uint64 next_ticket = ticket + 1;
2667 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2668 ticket, next_ticket)) {
2669 KMP_FSYNC_ACQUIRED(lck);
2670 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2672 lck->lk.now_serving = ticket;
2690 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2692 char const *
const func =
"omp_test_lock";
2693 if ( lck->lk.initialized != lck ) {
2694 KMP_FATAL( LockIsUninitialized, func );
2696 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2697 KMP_FATAL( LockNestableUsedAsSimple, func );
2700 int retval = __kmp_test_drdpa_lock( lck, gtid );
2703 lck->lk.owner_id = gtid + 1;
2709 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2716 kmp_uint64 ticket = lck->lk.now_serving + 1;
2717 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2718 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2719 TCR_PTR(lck->lk.polls);
2720 kmp_uint64 mask = TCR_8(lck->lk.mask);
2721 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2723 KMP_FSYNC_RELEASING(lck);
2724 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2725 return KMP_LOCK_RELEASED;
2729 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2731 char const *
const func =
"omp_unset_lock";
2733 if ( lck->lk.initialized != lck ) {
2734 KMP_FATAL( LockIsUninitialized, func );
2736 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2737 KMP_FATAL( LockNestableUsedAsSimple, func );
2739 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2740 KMP_FATAL( LockUnsettingFree, func );
2742 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2743 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2744 KMP_FATAL( LockUnsettingSetByAnother, func );
2746 lck->lk.owner_id = 0;
2747 return __kmp_release_drdpa_lock( lck, gtid );
2751 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2753 lck->lk.location = NULL;
2755 lck->lk.num_polls = 1;
2756 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2757 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2758 lck->lk.cleanup_ticket = 0;
2759 lck->lk.old_polls = NULL;
2760 lck->lk.next_ticket = 0;
2761 lck->lk.now_serving = 0;
2762 lck->lk.owner_id = 0;
2763 lck->lk.depth_locked = -1;
2764 lck->lk.initialized = lck;
2766 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2770 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2772 __kmp_init_drdpa_lock( lck );
2776 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2778 lck->lk.initialized = NULL;
2779 lck->lk.location = NULL;
2780 if (lck->lk.polls != NULL) {
2781 __kmp_free((
void *)lck->lk.polls);
2782 lck->lk.polls = NULL;
2784 if (lck->lk.old_polls != NULL) {
2785 __kmp_free((
void *)lck->lk.old_polls);
2786 lck->lk.old_polls = NULL;
2789 lck->lk.num_polls = 0;
2790 lck->lk.cleanup_ticket = 0;
2791 lck->lk.next_ticket = 0;
2792 lck->lk.now_serving = 0;
2793 lck->lk.owner_id = 0;
2794 lck->lk.depth_locked = -1;
2798 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2800 char const *
const func =
"omp_destroy_lock";
2801 if ( lck->lk.initialized != lck ) {
2802 KMP_FATAL( LockIsUninitialized, func );
2804 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2805 KMP_FATAL( LockNestableUsedAsSimple, func );
2807 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2808 KMP_FATAL( LockStillOwned, func );
2810 __kmp_destroy_drdpa_lock( lck );
2819 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2821 KMP_DEBUG_ASSERT( gtid >= 0 );
2823 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2824 lck->lk.depth_locked += 1;
2827 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2829 lck->lk.depth_locked = 1;
2831 lck->lk.owner_id = gtid + 1;
2836 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2838 char const *
const func =
"omp_set_nest_lock";
2839 if ( lck->lk.initialized != lck ) {
2840 KMP_FATAL( LockIsUninitialized, func );
2842 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2843 KMP_FATAL( LockSimpleUsedAsNestable, func );
2845 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2849 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2853 KMP_DEBUG_ASSERT( gtid >= 0 );
2855 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2856 retval = ++lck->lk.depth_locked;
2858 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2863 retval = lck->lk.depth_locked = 1;
2865 lck->lk.owner_id = gtid + 1;
2871 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2873 char const *
const func =
"omp_test_nest_lock";
2874 if ( lck->lk.initialized != lck ) {
2875 KMP_FATAL( LockIsUninitialized, func );
2877 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2878 KMP_FATAL( LockSimpleUsedAsNestable, func );
2880 return __kmp_test_nested_drdpa_lock( lck, gtid );
2884 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2886 KMP_DEBUG_ASSERT( gtid >= 0 );
2889 if ( --(lck->lk.depth_locked) == 0 ) {
2891 lck->lk.owner_id = 0;
2892 __kmp_release_drdpa_lock( lck, gtid );
2893 return KMP_LOCK_RELEASED;
2895 return KMP_LOCK_STILL_HELD;
2899 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2901 char const *
const func =
"omp_unset_nest_lock";
2903 if ( lck->lk.initialized != lck ) {
2904 KMP_FATAL( LockIsUninitialized, func );
2906 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2907 KMP_FATAL( LockSimpleUsedAsNestable, func );
2909 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2910 KMP_FATAL( LockUnsettingFree, func );
2912 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2913 KMP_FATAL( LockUnsettingSetByAnother, func );
2915 return __kmp_release_nested_drdpa_lock( lck, gtid );
2919 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2921 __kmp_init_drdpa_lock( lck );
2922 lck->lk.depth_locked = 0;
2926 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2928 __kmp_init_nested_drdpa_lock( lck );
2932 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2934 __kmp_destroy_drdpa_lock( lck );
2935 lck->lk.depth_locked = 0;
2939 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2941 char const *
const func =
"omp_destroy_nest_lock";
2942 if ( lck->lk.initialized != lck ) {
2943 KMP_FATAL( LockIsUninitialized, func );
2945 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2946 KMP_FATAL( LockSimpleUsedAsNestable, func );
2948 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2949 KMP_FATAL( LockStillOwned, func );
2951 __kmp_destroy_nested_drdpa_lock( lck );
2960 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
2962 return lck == lck->lk.initialized;
2966 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
2968 return lck->lk.location;
2972 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
2974 lck->lk.location = loc;
2977 static kmp_lock_flags_t
2978 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
2980 return lck->lk.flags;
2984 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
2986 lck->lk.flags = flags;
2989 #if KMP_USE_DYNAMIC_LOCK 2993 typedef enum kmp_lock_hint_t {
2994 kmp_lock_hint_none = 0,
2995 kmp_lock_hint_contended,
2996 kmp_lock_hint_uncontended,
2997 kmp_lock_hint_nonspeculative,
2998 kmp_lock_hint_speculative,
2999 kmp_lock_hint_adaptive,
3004 #define expand_init_lock(l, a) \ 3005 static void init_##l##_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq) { \ 3006 *lck = DYNA_LOCK_FREE(l); \ 3007 KA_TRACE(20, ("Initialized direct lock, tag = %x\n", *lck)); \ 3009 FOREACH_D_LOCK(expand_init_lock, 0)
3010 #undef expand_init_lock 3016 # define machine_pause() _mm_delay_32(10) // TODO: find the right argument 3018 # define machine_pause() _mm_pause() 3020 #define HLE_ACQUIRE ".byte 0xf2;" 3021 #define HLE_RELEASE ".byte 0xf3;" 3023 static inline kmp_uint32
3024 swap4(kmp_uint32
volatile *p, kmp_uint32 v)
3026 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" 3034 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
3040 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3043 if (swap4(lck, DYNA_LOCK_BUSY(1, hle)) != DYNA_LOCK_FREE(hle)) {
3046 while (*(kmp_uint32
volatile *)lck != DYNA_LOCK_FREE(hle)) {
3047 for (
int i = delay; i != 0; --i)
3049 delay = ((delay << 1) | 1) & 7;
3051 }
while (swap4(lck, DYNA_LOCK_BUSY(1, hle)) != DYNA_LOCK_FREE(hle));
3056 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3058 __kmp_acquire_hle_lock(lck, gtid);
3062 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3064 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 3066 :
"r"(DYNA_LOCK_FREE(hle))
3071 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3073 __kmp_release_hle_lock(lck, gtid);
3077 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3079 return swap4(lck, DYNA_LOCK_BUSY(1, hle)) == DYNA_LOCK_FREE(hle);
3083 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3085 return __kmp_test_hle_lock(lck, gtid);
3088 #endif // DYNA_HAS_HLE 3091 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
3092 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
3093 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3094 static void __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3095 static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3096 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3097 static void __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3098 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3104 #define expand_func0(l, op) 0,op##_##l##_##lock, 3105 void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
3106 = { __kmp_init_indirect_lock, 0, FOREACH_D_LOCK(expand_func0, init) };
3108 #define expand_func1(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_##lock, 3109 void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *)
3110 = { __kmp_destroy_indirect_lock, 0, FOREACH_D_LOCK(expand_func1, destroy) };
3113 #define expand_func2(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock, 3114 #define expand_func2c(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3115 static void (*direct_set_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3116 = { { __kmp_set_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, acquire) },
3117 { __kmp_set_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, acquire) } };
3118 static void (*direct_unset_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3119 = { { __kmp_unset_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, release) },
3120 { __kmp_unset_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, release) } };
3122 #define expand_func3(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock, 3123 #define expand_func3c(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3124 static int (*direct_test_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3125 = { { __kmp_test_indirect_lock, 0, FOREACH_D_LOCK(expand_func3, test) },
3126 { __kmp_test_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func3c, test) } };
3129 void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3130 void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3131 int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3136 #define expand_func4(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock, 3137 void (*__kmp_indirect_init_ops[])(kmp_user_lock_p)
3138 = { FOREACH_I_LOCK(expand_func4, init) };
3139 void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p)
3140 = { FOREACH_I_LOCK(expand_func4, destroy) };
3143 #define expand_func5(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3144 #define expand_func5c(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3145 static void (*indirect_set_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3146 = { { FOREACH_I_LOCK(expand_func5, acquire) },
3147 { FOREACH_I_LOCK(expand_func5c, acquire) } };
3148 static void (*indirect_unset_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3149 = { { FOREACH_I_LOCK(expand_func5, release) },
3150 { FOREACH_I_LOCK(expand_func5c, release) } };
3152 #define expand_func6(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3153 #define expand_func6c(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3154 static int (*indirect_test_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3155 = { { FOREACH_I_LOCK(expand_func6, test) },
3156 { FOREACH_I_LOCK(expand_func6c, test) } };
3159 void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32) = 0;
3160 void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32) = 0;
3161 int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32) = 0;
3164 kmp_indirect_lock_t **__kmp_indirect_lock_table;
3165 kmp_lock_index_t __kmp_indirect_lock_table_size;
3166 kmp_lock_index_t __kmp_indirect_lock_table_next;
3169 static kmp_uint32 __kmp_indirect_lock_size[DYNA_NUM_I_LOCKS] = {
3170 sizeof(kmp_ticket_lock_t),
sizeof(kmp_queuing_lock_t),
3171 #if KMP_USE_ADAPTIVE_LOCKS 3172 sizeof(kmp_adaptive_lock_t),
3174 sizeof(kmp_drdpa_lock_t),
3175 sizeof(kmp_tas_lock_t),
3177 sizeof(kmp_futex_lock_t),
3179 sizeof(kmp_ticket_lock_t),
sizeof(kmp_queuing_lock_t),
3180 sizeof(kmp_drdpa_lock_t)
3184 void (*__kmp_indirect_set_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *) = { 0 };
3185 void (*__kmp_indirect_set_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t) = { 0 };
3186 const ident_t * (*__kmp_indirect_get_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3187 kmp_lock_flags_t (*__kmp_indirect_get_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3190 static kmp_indirect_lock_t * __kmp_indirect_lock_pool[DYNA_NUM_I_LOCKS] = { 0 };
3194 __kmp_insert_indirect_lock(kmp_indirect_lock_t *lck)
3196 kmp_lock_index_t next = __kmp_indirect_lock_table_next;
3198 if (next >= __kmp_indirect_lock_table_size) {
3200 kmp_lock_index_t size = __kmp_indirect_lock_table_size;
3201 kmp_indirect_lock_t **old_table = __kmp_indirect_lock_table;
3202 __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(2*next*
sizeof(kmp_indirect_lock_t *));
3203 KMP_MEMCPY(__kmp_indirect_lock_table, old_table, next*
sizeof(kmp_indirect_lock_t *));
3204 __kmp_free(old_table);
3205 __kmp_indirect_lock_table_size = 2*next;
3208 __kmp_indirect_lock_table[next] = lck;
3209 __kmp_indirect_lock_table_next++;
3214 kmp_indirect_lock_t *
3215 __kmp_allocate_indirect_lock(
void **user_lock, kmp_int32 gtid, kmp_indirect_locktag_t tag)
3217 kmp_indirect_lock_t *lck;
3218 kmp_lock_index_t idx;
3220 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3222 if (__kmp_indirect_lock_pool[tag] != NULL) {
3223 lck = __kmp_indirect_lock_pool[tag];
3224 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3225 idx = lck->lock->pool.index;
3226 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3228 lck = (kmp_indirect_lock_t *)__kmp_allocate(
sizeof(kmp_indirect_lock_t));
3229 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3230 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3231 idx = __kmp_insert_indirect_lock(lck);
3234 __kmp_release_lock(&__kmp_global_lock, gtid);
3238 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3239 *((kmp_lock_index_t *)user_lock) = idx << 1;
3241 *((kmp_indirect_lock_t **)user_lock) = lck;
3248 static __forceinline
3249 kmp_indirect_lock_t *
3250 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func)
3252 if (__kmp_env_consistency_check) {
3253 kmp_indirect_lock_t *lck = NULL;
3254 if (user_lock == NULL) {
3255 KMP_FATAL(LockIsUninitialized, func);
3257 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3258 kmp_lock_index_t idx = DYNA_EXTRACT_I_INDEX(user_lock);
3259 if (idx < 0 || idx >= __kmp_indirect_lock_table_size) {
3260 KMP_FATAL(LockIsUninitialized, func);
3262 lck = __kmp_indirect_lock_table[idx];
3264 lck = *((kmp_indirect_lock_t **)user_lock);
3267 KMP_FATAL(LockIsUninitialized, func);
3271 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3272 return __kmp_indirect_lock_table[DYNA_EXTRACT_I_INDEX(user_lock)];
3274 return *((kmp_indirect_lock_t **)user_lock);
3280 __kmp_init_indirect_lock(kmp_dyna_lock_t * lock, kmp_dyna_lockseq_t seq)
3282 #if KMP_USE_ADAPTIVE_LOCKS 3283 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3284 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3285 seq = lockseq_queuing;
3288 kmp_indirect_locktag_t tag = DYNA_GET_I_TAG(seq);
3289 kmp_indirect_lock_t *l = __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3290 DYNA_I_LOCK_FUNC(l, init)(l->lock);
3291 KA_TRACE(20, (
"__kmp_init_indirect_lock: initialized indirect lock, tag = %x\n", l->type));
3295 __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock)
3297 kmp_uint32 gtid = __kmp_entry_gtid();
3298 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3299 DYNA_I_LOCK_FUNC(l, destroy)(l->lock);
3300 kmp_indirect_locktag_t tag = l->type;
3302 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3305 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3306 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3307 l->lock->pool.index = DYNA_EXTRACT_I_INDEX(lock);
3309 __kmp_indirect_lock_pool[tag] = l;
3311 __kmp_release_lock(&__kmp_global_lock, gtid);
3315 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3317 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3318 DYNA_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3322 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3324 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3325 DYNA_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3329 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3331 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3332 return DYNA_I_LOCK_FUNC(l, test)(l->lock, gtid);
3336 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3338 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3339 DYNA_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3343 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3345 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3346 DYNA_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3350 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3352 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3353 return DYNA_I_LOCK_FUNC(l, test)(l->lock, gtid);
3356 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3360 __kmp_init_lock_hinted(
void **lock,
int hint)
3362 kmp_dyna_lockseq_t seq;
3364 case kmp_lock_hint_uncontended:
3367 case kmp_lock_hint_speculative:
3374 case kmp_lock_hint_adaptive:
3375 #if KMP_USE_ADAPTIVE_LOCKS 3376 seq = lockseq_adaptive;
3378 seq = lockseq_queuing;
3382 case kmp_lock_hint_contended:
3383 case kmp_lock_hint_nonspeculative:
3385 seq = lockseq_queuing;
3388 if (DYNA_IS_D_LOCK(seq)) {
3389 DYNA_INIT_D_LOCK(lock, seq);
3391 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
3394 DYNA_INIT_I_LOCK(lock, seq);
3396 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(lock);
3397 __kmp_itt_lock_creating(ilk->lock, NULL);
3404 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
3408 case lockseq_nested_tas:
3409 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3412 case lockseq_nested_futex:
3413 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3415 case lockseq_ticket:
3416 case lockseq_nested_ticket:
3417 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3418 case lockseq_queuing:
3419 case lockseq_nested_queuing:
3420 #if KMP_USE_ADAPTIVE_LOCKS 3421 case lockseq_adaptive:
3422 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3425 case lockseq_nested_drdpa:
3426 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3435 __kmp_init_nest_lock_hinted(
void **lock,
int hint)
3437 kmp_dyna_lockseq_t seq;
3439 case kmp_lock_hint_uncontended:
3440 seq = lockseq_nested_tas;
3443 case kmp_lock_hint_contended:
3444 case kmp_lock_hint_nonspeculative:
3446 seq = lockseq_nested_queuing;
3449 DYNA_INIT_I_LOCK(lock, seq);
3451 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(lock);
3452 __kmp_itt_lock_creating(ilk->lock, NULL);
3458 __kmp_init_indirect_lock_table()
3460 __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *)*1024);
3461 __kmp_indirect_lock_table_size = 1024;
3462 __kmp_indirect_lock_table_next = 0;
3465 #if KMP_USE_ADAPTIVE_LOCKS 3466 # define init_lock_func(table, expand) { \ 3467 table[locktag_ticket] = expand(ticket); \ 3468 table[locktag_queuing] = expand(queuing); \ 3469 table[locktag_adaptive] = expand(queuing); \ 3470 table[locktag_drdpa] = expand(drdpa); \ 3471 table[locktag_nested_ticket] = expand(ticket); \ 3472 table[locktag_nested_queuing] = expand(queuing); \ 3473 table[locktag_nested_drdpa] = expand(drdpa); \ 3476 # define init_lock_func(table, expand) { \ 3477 table[locktag_ticket] = expand(ticket); \ 3478 table[locktag_queuing] = expand(queuing); \ 3479 table[locktag_drdpa] = expand(drdpa); \ 3480 table[locktag_nested_ticket] = expand(ticket); \ 3481 table[locktag_nested_queuing] = expand(queuing); \ 3482 table[locktag_nested_drdpa] = expand(drdpa); \ 3484 #endif // KMP_USE_ADAPTIVE_LOCKS 3488 __kmp_init_dynamic_user_locks()
3491 int offset = (__kmp_env_consistency_check)? 1: 0;
3492 __kmp_direct_set_ops = direct_set_tab[offset];
3493 __kmp_direct_unset_ops = direct_unset_tab[offset];
3494 __kmp_direct_test_ops = direct_test_tab[offset];
3495 __kmp_indirect_set_ops = indirect_set_tab[offset];
3496 __kmp_indirect_unset_ops = indirect_unset_tab[offset];
3497 __kmp_indirect_test_ops = indirect_test_tab[offset];
3498 __kmp_init_indirect_lock_table();
3502 #define expand_func(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location 3503 init_lock_func(__kmp_indirect_set_location, expand_func);
3505 #define expand_func(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags 3506 init_lock_func(__kmp_indirect_set_flags, expand_func);
3508 #define expand_func(l) (const ident_t * (*)(kmp_user_lock_p))__kmp_get_##l##_lock_location 3509 init_lock_func(__kmp_indirect_get_location, expand_func);
3511 #define expand_func(l) (kmp_lock_flags_t (*)(kmp_user_lock_p))__kmp_get_##l##_lock_flags 3512 init_lock_func(__kmp_indirect_get_flags, expand_func);
3515 __kmp_init_user_locks = TRUE;
3520 __kmp_cleanup_indirect_user_locks()
3526 for (k = 0; k < DYNA_NUM_I_LOCKS; ++k) {
3527 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3529 kmp_indirect_lock_t *ll = l;
3530 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3531 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3532 __kmp_indirect_lock_table[ll->lock->pool.index] = NULL;
3534 __kmp_free(ll->lock);
3539 for (i = 0; i < __kmp_indirect_lock_table_next; i++) {
3540 kmp_indirect_lock_t *l = __kmp_indirect_lock_table[i];
3543 DYNA_I_LOCK_FUNC(l, destroy)(l->lock);
3544 __kmp_free(l->lock);
3549 __kmp_free(__kmp_indirect_lock_table);
3551 __kmp_init_user_locks = FALSE;
3554 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3555 int __kmp_num_locks_in_block = 1;
3557 #else // KMP_USE_DYNAMIC_LOCK 3566 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3568 size_t __kmp_base_user_lock_size = 0;
3569 size_t __kmp_user_lock_size = 0;
3571 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3572 void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3574 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3575 int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3576 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3577 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3578 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3579 void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3581 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3582 int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3583 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3584 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3586 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3587 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3588 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3589 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3590 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3592 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3594 switch ( user_lock_kind ) {
3600 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3601 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3603 __kmp_get_user_lock_owner_ =
3604 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3605 ( &__kmp_get_tas_lock_owner );
3607 if ( __kmp_env_consistency_check ) {
3608 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3609 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3612 KMP_BIND_USER_LOCK(tas);
3613 KMP_BIND_NESTED_USER_LOCK(tas);
3616 __kmp_destroy_user_lock_ =
3617 ( void ( * )( kmp_user_lock_p ) )
3618 ( &__kmp_destroy_tas_lock );
3620 __kmp_is_user_lock_initialized_ =
3621 ( int ( * )( kmp_user_lock_p ) ) NULL;
3623 __kmp_get_user_lock_location_ =
3624 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3626 __kmp_set_user_lock_location_ =
3627 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3629 __kmp_get_user_lock_flags_ =
3630 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3632 __kmp_set_user_lock_flags_ =
3633 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3637 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) 3640 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3641 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3643 __kmp_get_user_lock_owner_ =
3644 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3645 ( &__kmp_get_futex_lock_owner );
3647 if ( __kmp_env_consistency_check ) {
3648 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3649 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3652 KMP_BIND_USER_LOCK(futex);
3653 KMP_BIND_NESTED_USER_LOCK(futex);
3656 __kmp_destroy_user_lock_ =
3657 ( void ( * )( kmp_user_lock_p ) )
3658 ( &__kmp_destroy_futex_lock );
3660 __kmp_is_user_lock_initialized_ =
3661 ( int ( * )( kmp_user_lock_p ) ) NULL;
3663 __kmp_get_user_lock_location_ =
3664 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3666 __kmp_set_user_lock_location_ =
3667 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3669 __kmp_get_user_lock_flags_ =
3670 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3672 __kmp_set_user_lock_flags_ =
3673 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3677 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) 3680 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3681 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3683 __kmp_get_user_lock_owner_ =
3684 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3685 ( &__kmp_get_ticket_lock_owner );
3687 if ( __kmp_env_consistency_check ) {
3688 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3689 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3692 KMP_BIND_USER_LOCK(ticket);
3693 KMP_BIND_NESTED_USER_LOCK(ticket);
3696 __kmp_destroy_user_lock_ =
3697 ( void ( * )( kmp_user_lock_p ) )
3698 ( &__kmp_destroy_ticket_lock );
3700 __kmp_is_user_lock_initialized_ =
3701 ( int ( * )( kmp_user_lock_p ) )
3702 ( &__kmp_is_ticket_lock_initialized );
3704 __kmp_get_user_lock_location_ =
3705 (
const ident_t * ( * )( kmp_user_lock_p ) )
3706 ( &__kmp_get_ticket_lock_location );
3708 __kmp_set_user_lock_location_ =
3709 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3710 ( &__kmp_set_ticket_lock_location );
3712 __kmp_get_user_lock_flags_ =
3713 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3714 ( &__kmp_get_ticket_lock_flags );
3716 __kmp_set_user_lock_flags_ =
3717 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3718 ( &__kmp_set_ticket_lock_flags );
3723 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3724 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3726 __kmp_get_user_lock_owner_ =
3727 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3728 ( &__kmp_get_queuing_lock_owner );
3730 if ( __kmp_env_consistency_check ) {
3731 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3732 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3735 KMP_BIND_USER_LOCK(queuing);
3736 KMP_BIND_NESTED_USER_LOCK(queuing);
3739 __kmp_destroy_user_lock_ =
3740 ( void ( * )( kmp_user_lock_p ) )
3741 ( &__kmp_destroy_queuing_lock );
3743 __kmp_is_user_lock_initialized_ =
3744 ( int ( * )( kmp_user_lock_p ) )
3745 ( &__kmp_is_queuing_lock_initialized );
3747 __kmp_get_user_lock_location_ =
3748 (
const ident_t * ( * )( kmp_user_lock_p ) )
3749 ( &__kmp_get_queuing_lock_location );
3751 __kmp_set_user_lock_location_ =
3752 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3753 ( &__kmp_set_queuing_lock_location );
3755 __kmp_get_user_lock_flags_ =
3756 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3757 ( &__kmp_get_queuing_lock_flags );
3759 __kmp_set_user_lock_flags_ =
3760 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3761 ( &__kmp_set_queuing_lock_flags );
3765 #if KMP_USE_ADAPTIVE_LOCKS 3767 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3768 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3770 __kmp_get_user_lock_owner_ =
3771 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3772 ( &__kmp_get_queuing_lock_owner );
3774 if ( __kmp_env_consistency_check ) {
3775 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3778 KMP_BIND_USER_LOCK(adaptive);
3781 __kmp_destroy_user_lock_ =
3782 ( void ( * )( kmp_user_lock_p ) )
3783 ( &__kmp_destroy_adaptive_lock );
3785 __kmp_is_user_lock_initialized_ =
3786 ( int ( * )( kmp_user_lock_p ) )
3787 ( &__kmp_is_queuing_lock_initialized );
3789 __kmp_get_user_lock_location_ =
3790 (
const ident_t * ( * )( kmp_user_lock_p ) )
3791 ( &__kmp_get_queuing_lock_location );
3793 __kmp_set_user_lock_location_ =
3794 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3795 ( &__kmp_set_queuing_lock_location );
3797 __kmp_get_user_lock_flags_ =
3798 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3799 ( &__kmp_get_queuing_lock_flags );
3801 __kmp_set_user_lock_flags_ =
3802 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3803 ( &__kmp_set_queuing_lock_flags );
3807 #endif // KMP_USE_ADAPTIVE_LOCKS 3810 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3811 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3813 __kmp_get_user_lock_owner_ =
3814 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3815 ( &__kmp_get_drdpa_lock_owner );
3817 if ( __kmp_env_consistency_check ) {
3818 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3819 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3822 KMP_BIND_USER_LOCK(drdpa);
3823 KMP_BIND_NESTED_USER_LOCK(drdpa);
3826 __kmp_destroy_user_lock_ =
3827 ( void ( * )( kmp_user_lock_p ) )
3828 ( &__kmp_destroy_drdpa_lock );
3830 __kmp_is_user_lock_initialized_ =
3831 ( int ( * )( kmp_user_lock_p ) )
3832 ( &__kmp_is_drdpa_lock_initialized );
3834 __kmp_get_user_lock_location_ =
3835 (
const ident_t * ( * )( kmp_user_lock_p ) )
3836 ( &__kmp_get_drdpa_lock_location );
3838 __kmp_set_user_lock_location_ =
3839 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3840 ( &__kmp_set_drdpa_lock_location );
3842 __kmp_get_user_lock_flags_ =
3843 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3844 ( &__kmp_get_drdpa_lock_flags );
3846 __kmp_set_user_lock_flags_ =
3847 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3848 ( &__kmp_set_drdpa_lock_flags );
3858 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3859 kmp_user_lock_p __kmp_lock_pool = NULL;
3862 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3863 int __kmp_num_locks_in_block = 1;
3865 static kmp_lock_index_t
3866 __kmp_lock_table_insert( kmp_user_lock_p lck )
3869 kmp_lock_index_t index;
3870 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3871 kmp_lock_index_t size;
3872 kmp_user_lock_p *table;
3874 if ( __kmp_user_lock_table.allocated == 0 ) {
3878 size = __kmp_user_lock_table.allocated * 2;
3880 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3881 KMP_MEMCPY( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3882 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3887 __kmp_user_lock_table.table = table;
3888 __kmp_user_lock_table.allocated = size;
3890 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3891 index = __kmp_user_lock_table.used;
3892 __kmp_user_lock_table.table[ index ] = lck;
3893 ++ __kmp_user_lock_table.used;
3897 static kmp_user_lock_p
3898 __kmp_lock_block_allocate()
3901 static int last_index = 0;
3902 if ( ( last_index >= __kmp_num_locks_in_block )
3903 || ( __kmp_lock_blocks == NULL ) ) {
3907 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3908 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3909 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
3911 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
3912 new_block->next_block = __kmp_lock_blocks;
3913 new_block->locks = (
void *)buffer;
3916 __kmp_lock_blocks = new_block;
3918 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
3919 [ last_index * __kmp_user_lock_size ] ) );
3929 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3930 kmp_lock_flags_t flags )
3932 kmp_user_lock_p lck;
3933 kmp_lock_index_t index;
3934 KMP_DEBUG_ASSERT( user_lock );
3936 __kmp_acquire_lock( &__kmp_global_lock, gtid );
3938 if ( __kmp_lock_pool == NULL ) {
3940 if ( __kmp_num_locks_in_block <= 1 ) {
3941 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
3944 lck = __kmp_lock_block_allocate();
3949 index = __kmp_lock_table_insert( lck );
3953 lck = __kmp_lock_pool;
3954 index = __kmp_lock_pool->pool.index;
3955 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3962 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3963 * ( (kmp_lock_index_t *) user_lock ) = index;
3966 * ( (kmp_user_lock_p *) user_lock ) = lck;
3970 __kmp_set_user_lock_flags( lck, flags );
3972 __kmp_release_lock( & __kmp_global_lock, gtid );
3979 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
3981 KMP_DEBUG_ASSERT( user_lock != NULL );
3982 KMP_DEBUG_ASSERT( lck != NULL );
3984 __kmp_acquire_lock( & __kmp_global_lock, gtid );
3986 lck->pool.next = __kmp_lock_pool;
3987 __kmp_lock_pool = lck;
3988 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3989 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
3990 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
3991 lck->pool.index = index;
3994 __kmp_release_lock( & __kmp_global_lock, gtid );
3998 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
4000 kmp_user_lock_p lck = NULL;
4002 if ( __kmp_env_consistency_check ) {
4003 if ( user_lock == NULL ) {
4004 KMP_FATAL( LockIsUninitialized, func );
4008 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4009 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
4010 if ( __kmp_env_consistency_check ) {
4011 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
4012 KMP_FATAL( LockIsUninitialized, func );
4015 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
4016 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4017 lck = __kmp_user_lock_table.table[index];
4020 lck = *( (kmp_user_lock_p *)user_lock );
4023 if ( __kmp_env_consistency_check ) {
4024 if ( lck == NULL ) {
4025 KMP_FATAL( LockIsUninitialized, func );
4033 __kmp_cleanup_user_locks(
void )
4040 __kmp_lock_pool = NULL;
4042 #define IS_CRITICAL(lck) \ 4043 ( ( __kmp_get_user_lock_flags_ != NULL ) && \ 4044 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) ) 4074 while ( __kmp_user_lock_table.used > 1 ) {
4081 kmp_user_lock_p lck = __kmp_user_lock_table.table[
4082 --__kmp_user_lock_table.used ];
4084 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
4085 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
4092 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
4093 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
4095 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
4096 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.line );
4097 __kmp_str_loc_free( &str_loc);
4101 if ( IS_CRITICAL( lck ) ) {
4102 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
4105 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
4113 __kmp_destroy_user_lock( lck );
4119 if ( __kmp_lock_blocks == NULL ) {
4129 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4130 __kmp_user_lock_table.table = NULL;
4131 __kmp_user_lock_table.allocated = 0;
4133 while ( table_ptr != NULL ) {
4138 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
4139 __kmp_free( table_ptr );
4146 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4147 __kmp_lock_blocks = NULL;
4149 while ( block_ptr != NULL ) {
4150 kmp_block_of_locks_t *next = block_ptr->next_block;
4151 __kmp_free( block_ptr->locks );
4158 TCW_4(__kmp_init_user_locks, FALSE);
4161 #endif // KMP_USE_DYNAMIC_LOCK