23 #include "kmp_debug.h" 33 #define KMP_PAD(type, sz) (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1)) 34 #define KMP_GTID_DNE (-2) 55 #if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT) 56 # define OMP_LOCK_T_SIZE sizeof(int) 57 # define OMP_NEST_LOCK_T_SIZE sizeof(void *) 59 # define OMP_LOCK_T_SIZE sizeof(void *) 60 # define OMP_NEST_LOCK_T_SIZE sizeof(void *) 68 #define OMP_CRITICAL_SIZE sizeof(void *) 69 #define INTEL_CRITICAL_SIZE 32 74 typedef kmp_uint32 kmp_lock_flags_t;
76 #define kmp_lf_critical_section 1 81 typedef kmp_uint32 kmp_lock_index_t;
87 struct kmp_lock_pool {
88 union kmp_user_lock *next;
89 kmp_lock_index_t index;
92 typedef struct kmp_lock_pool kmp_lock_pool_t;
95 extern void __kmp_validate_locks(
void );
139 struct kmp_base_tas_lock {
140 volatile kmp_int32 poll;
142 kmp_int32 depth_locked;
145 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
148 kmp_base_tas_lock_t lk;
149 kmp_lock_pool_t pool;
154 typedef union kmp_tas_lock kmp_tas_lock_t;
160 #define KMP_TAS_LOCK_INITIALIZER( lock ) { { 0, 0 } } 162 extern void __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
163 extern int __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
164 extern int __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
165 extern void __kmp_init_tas_lock( kmp_tas_lock_t *lck );
166 extern void __kmp_destroy_tas_lock( kmp_tas_lock_t *lck );
168 extern void __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
169 extern int __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
170 extern int __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
171 extern void __kmp_init_nested_tas_lock( kmp_tas_lock_t *lck );
172 extern void __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck );
174 #define KMP_LOCK_RELEASED 1 175 #define KMP_LOCK_STILL_HELD 0 178 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 192 struct kmp_base_futex_lock {
193 volatile kmp_int32 poll;
196 kmp_int32 depth_locked;
199 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
201 union kmp_futex_lock {
202 kmp_base_futex_lock_t lk;
203 kmp_lock_pool_t pool;
208 typedef union kmp_futex_lock kmp_futex_lock_t;
214 #define KMP_FUTEX_LOCK_INITIALIZER( lock ) { { 0, 0 } } 216 extern void __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
217 extern int __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
218 extern int __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
219 extern void __kmp_init_futex_lock( kmp_futex_lock_t *lck );
220 extern void __kmp_destroy_futex_lock( kmp_futex_lock_t *lck );
222 extern void __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
223 extern int __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
224 extern int __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
225 extern void __kmp_init_nested_futex_lock( kmp_futex_lock_t *lck );
226 extern void __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck );
228 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 235 struct kmp_base_ticket_lock {
237 volatile union kmp_ticket_lock * initialized;
239 volatile kmp_uint32 next_ticket;
240 volatile kmp_uint32 now_serving;
241 volatile kmp_int32 owner_id;
242 kmp_int32 depth_locked;
243 kmp_lock_flags_t flags;
246 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
248 union KMP_ALIGN_CACHE kmp_ticket_lock {
249 kmp_base_ticket_lock_t lk;
250 kmp_lock_pool_t pool;
252 char lk_pad[ KMP_PAD( kmp_base_ticket_lock_t, CACHE_LINE ) ];
255 typedef union kmp_ticket_lock kmp_ticket_lock_t;
262 #define KMP_TICKET_LOCK_INITIALIZER( lock ) { { (kmp_ticket_lock_t *) & (lock), NULL, 0, 0, 0, -1 } } 264 extern void __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
265 extern int __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
266 extern int __kmp_test_ticket_lock_with_cheks( kmp_ticket_lock_t *lck, kmp_int32 gtid );
267 extern int __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
268 extern void __kmp_init_ticket_lock( kmp_ticket_lock_t *lck );
269 extern void __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck );
271 extern void __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
272 extern int __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
273 extern int __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
274 extern void __kmp_init_nested_ticket_lock( kmp_ticket_lock_t *lck );
275 extern void __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck );
282 #if KMP_USE_ADAPTIVE_LOCKS 284 struct kmp_adaptive_lock_info;
286 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
288 #if KMP_DEBUG_ADAPTIVE_LOCKS 290 struct kmp_adaptive_lock_statistics {
292 kmp_adaptive_lock_info_t * next;
293 kmp_adaptive_lock_info_t * prev;
296 kmp_uint32 successfulSpeculations;
297 kmp_uint32 hardFailedSpeculations;
298 kmp_uint32 softFailedSpeculations;
299 kmp_uint32 nonSpeculativeAcquires;
300 kmp_uint32 nonSpeculativeAcquireAttempts;
301 kmp_uint32 lemmingYields;
304 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
306 extern void __kmp_print_speculative_stats();
307 extern void __kmp_init_speculative_stats();
309 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 311 struct kmp_adaptive_lock_info
318 kmp_uint32
volatile badness;
319 kmp_uint32
volatile acquire_attempts;
321 kmp_uint32 max_badness;
322 kmp_uint32 max_soft_retries;
324 #if KMP_DEBUG_ADAPTIVE_LOCKS 325 kmp_adaptive_lock_statistics_t
volatile stats;
329 #endif // KMP_USE_ADAPTIVE_LOCKS 332 struct kmp_base_queuing_lock {
335 volatile union kmp_queuing_lock *initialized;
341 volatile kmp_int32 tail_id;
343 volatile kmp_int32 head_id;
346 volatile kmp_uint32 next_ticket;
347 volatile kmp_uint32 now_serving;
348 volatile kmp_int32 owner_id;
349 kmp_int32 depth_locked;
351 kmp_lock_flags_t flags;
354 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
356 KMP_BUILD_ASSERT( offsetof( kmp_base_queuing_lock_t, tail_id ) % 8 == 0 );
358 union KMP_ALIGN_CACHE kmp_queuing_lock {
359 kmp_base_queuing_lock_t lk;
360 kmp_lock_pool_t pool;
362 char lk_pad[ KMP_PAD( kmp_base_queuing_lock_t, CACHE_LINE ) ];
365 typedef union kmp_queuing_lock kmp_queuing_lock_t;
367 extern void __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
368 extern int __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
369 extern int __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
370 extern void __kmp_init_queuing_lock( kmp_queuing_lock_t *lck );
371 extern void __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck );
373 extern void __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
374 extern int __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
375 extern int __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
376 extern void __kmp_init_nested_queuing_lock( kmp_queuing_lock_t *lck );
377 extern void __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck );
379 #if KMP_USE_ADAPTIVE_LOCKS 384 struct kmp_base_adaptive_lock {
385 kmp_base_queuing_lock qlk;
386 KMP_ALIGN(CACHE_LINE)
387 kmp_adaptive_lock_info_t adaptive;
390 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
392 union KMP_ALIGN_CACHE kmp_adaptive_lock {
393 kmp_base_adaptive_lock_t lk;
394 kmp_lock_pool_t pool;
396 char lk_pad[ KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE) ];
398 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
400 # define GET_QLK_PTR(l) ((kmp_queuing_lock_t *) & (l)->lk.qlk) 402 #endif // KMP_USE_ADAPTIVE_LOCKS 408 struct kmp_base_drdpa_lock {
419 volatile union kmp_drdpa_lock * initialized;
421 volatile struct kmp_lock_poll {
424 volatile kmp_uint64 mask;
425 kmp_uint64 cleanup_ticket;
426 volatile struct kmp_lock_poll * old_polls;
427 kmp_uint32 num_polls;
435 volatile kmp_uint64 next_ticket;
454 kmp_uint64 now_serving;
455 volatile kmp_uint32 owner_id;
456 kmp_int32 depth_locked;
457 kmp_lock_flags_t flags;
460 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
462 union KMP_ALIGN_CACHE kmp_drdpa_lock {
463 kmp_base_drdpa_lock_t lk;
464 kmp_lock_pool_t pool;
466 char lk_pad[ KMP_PAD( kmp_base_drdpa_lock_t, CACHE_LINE ) ];
469 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
471 extern void __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
472 extern int __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
473 extern int __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
474 extern void __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck );
475 extern void __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck );
477 extern void __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
478 extern int __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
479 extern int __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
480 extern void __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
481 extern void __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
499 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
501 #define KMP_BOOTSTRAP_LOCK_INITIALIZER( lock ) KMP_TICKET_LOCK_INITIALIZER( (lock) ) 504 __kmp_acquire_bootstrap_lock( kmp_bootstrap_lock_t *lck )
506 __kmp_acquire_ticket_lock( lck, KMP_GTID_DNE );
510 __kmp_test_bootstrap_lock( kmp_bootstrap_lock_t *lck )
512 return __kmp_test_ticket_lock( lck, KMP_GTID_DNE );
516 __kmp_release_bootstrap_lock( kmp_bootstrap_lock_t *lck )
518 __kmp_release_ticket_lock( lck, KMP_GTID_DNE );
522 __kmp_init_bootstrap_lock( kmp_bootstrap_lock_t *lck )
524 __kmp_init_ticket_lock( lck );
528 __kmp_destroy_bootstrap_lock( kmp_bootstrap_lock_t *lck )
530 __kmp_destroy_ticket_lock( lck );
546 typedef kmp_ticket_lock_t kmp_lock_t;
549 __kmp_acquire_lock( kmp_lock_t *lck, kmp_int32 gtid )
551 __kmp_acquire_ticket_lock( lck, gtid );
555 __kmp_test_lock( kmp_lock_t *lck, kmp_int32 gtid )
557 return __kmp_test_ticket_lock( lck, gtid );
561 __kmp_release_lock( kmp_lock_t *lck, kmp_int32 gtid )
563 __kmp_release_ticket_lock( lck, gtid );
567 __kmp_init_lock( kmp_lock_t *lck )
569 __kmp_init_ticket_lock( lck );
573 __kmp_destroy_lock( kmp_lock_t *lck )
575 __kmp_destroy_ticket_lock( lck );
594 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 600 #if KMP_USE_ADAPTIVE_LOCKS 602 #endif // KMP_USE_ADAPTIVE_LOCKS 605 typedef enum kmp_lock_kind kmp_lock_kind_t;
607 extern kmp_lock_kind_t __kmp_user_lock_kind;
609 union kmp_user_lock {
611 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 612 kmp_futex_lock_t futex;
614 kmp_ticket_lock_t ticket;
615 kmp_queuing_lock_t queuing;
616 kmp_drdpa_lock_t drdpa;
617 #if KMP_USE_ADAPTIVE_LOCKS 618 kmp_adaptive_lock_t adaptive;
619 #endif // KMP_USE_ADAPTIVE_LOCKS 620 kmp_lock_pool_t pool;
623 typedef union kmp_user_lock *kmp_user_lock_p;
625 #if ! KMP_USE_DYNAMIC_LOCK 627 extern size_t __kmp_base_user_lock_size;
628 extern size_t __kmp_user_lock_size;
630 extern kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck );
632 static inline kmp_int32
633 __kmp_get_user_lock_owner( kmp_user_lock_p lck )
635 KMP_DEBUG_ASSERT( __kmp_get_user_lock_owner_ != NULL );
636 return ( *__kmp_get_user_lock_owner_ )( lck );
639 extern void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
641 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 643 #define __kmp_acquire_user_lock_with_checks(lck,gtid) \ 644 if (__kmp_user_lock_kind == lk_tas) { \ 645 if ( __kmp_env_consistency_check ) { \ 646 char const * const func = "omp_set_lock"; \ 647 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE ) \ 648 && lck->tas.lk.depth_locked != -1 ) { \ 649 KMP_FATAL( LockNestableUsedAsSimple, func ); \ 651 if ( ( gtid >= 0 ) && ( lck->tas.lk.poll - 1 == gtid ) ) { \ 652 KMP_FATAL( LockIsAlreadyOwned, func ); \ 655 if ( ( lck->tas.lk.poll != 0 ) || \ 656 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 658 KMP_FSYNC_PREPARE( lck ); \ 659 KMP_INIT_YIELD( spins ); \ 660 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 663 KMP_YIELD_SPIN( spins ); \ 665 while ( ( lck->tas.lk.poll != 0 ) || \ 666 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 667 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 670 KMP_YIELD_SPIN( spins ); \ 674 KMP_FSYNC_ACQUIRED( lck ); \ 676 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL ); \ 677 ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid ); \ 682 __kmp_acquire_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
684 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL );
685 ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid );
689 extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
691 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 693 #include "kmp_i18n.h" 694 extern int __kmp_env_consistency_check;
696 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
698 if ( __kmp_user_lock_kind == lk_tas ) {
699 if ( __kmp_env_consistency_check ) {
700 char const *
const func =
"omp_test_lock";
701 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
702 && lck->tas.lk.depth_locked != -1 ) {
703 KMP_FATAL( LockNestableUsedAsSimple, func );
706 return ( ( lck->tas.lk.poll == 0 ) &&
707 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
709 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
710 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
715 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
717 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
718 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
722 extern int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
725 __kmp_release_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
727 KMP_DEBUG_ASSERT( __kmp_release_user_lock_with_checks_ != NULL );
728 ( *__kmp_release_user_lock_with_checks_ ) ( lck, gtid );
731 extern void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck );
734 __kmp_init_user_lock_with_checks( kmp_user_lock_p lck )
736 KMP_DEBUG_ASSERT( __kmp_init_user_lock_with_checks_ != NULL );
737 ( *__kmp_init_user_lock_with_checks_ )( lck );
744 extern void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck );
747 __kmp_destroy_user_lock( kmp_user_lock_p lck )
749 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_ != NULL );
750 ( *__kmp_destroy_user_lock_ )( lck );
753 extern void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck );
756 __kmp_destroy_user_lock_with_checks( kmp_user_lock_p lck )
758 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_with_checks_ != NULL );
759 ( *__kmp_destroy_user_lock_with_checks_ )( lck );
762 extern void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
764 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 766 #define __kmp_acquire_nested_user_lock_with_checks(lck,gtid) \ 767 if (__kmp_user_lock_kind == lk_tas) { \ 768 if ( __kmp_env_consistency_check ) { \ 769 char const * const func = "omp_set_nest_lock"; \ 770 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE ) \ 771 && lck->tas.lk.depth_locked == -1 ) { \ 772 KMP_FATAL( LockSimpleUsedAsNestable, func ); \ 775 if ( lck->tas.lk.poll - 1 == gtid ) { \ 776 lck->tas.lk.depth_locked += 1; \ 778 if ( ( lck->tas.lk.poll != 0 ) || \ 779 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 781 KMP_FSYNC_PREPARE( lck ); \ 782 KMP_INIT_YIELD( spins ); \ 783 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 786 KMP_YIELD_SPIN( spins ); \ 788 while ( ( lck->tas.lk.poll != 0 ) || \ 789 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 790 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 793 KMP_YIELD_SPIN( spins ); \ 797 lck->tas.lk.depth_locked = 1; \ 799 KMP_FSYNC_ACQUIRED( lck ); \ 801 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL ); \ 802 ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid ); \ 807 __kmp_acquire_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
809 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL );
810 ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid );
814 extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
816 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 818 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
820 if ( __kmp_user_lock_kind == lk_tas ) {
822 if ( __kmp_env_consistency_check ) {
823 char const *
const func =
"omp_test_nest_lock";
824 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE )
825 && lck->tas.lk.depth_locked == -1 ) {
826 KMP_FATAL( LockSimpleUsedAsNestable, func );
829 KMP_DEBUG_ASSERT( gtid >= 0 );
830 if ( lck->tas.lk.poll - 1 == gtid ) {
831 return ++lck->tas.lk.depth_locked;
833 retval = ( ( lck->tas.lk.poll == 0 ) &&
834 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
837 lck->tas.lk.depth_locked = 1;
841 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
842 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
847 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
849 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
850 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
854 extern int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
857 __kmp_release_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
859 KMP_DEBUG_ASSERT( __kmp_release_nested_user_lock_with_checks_ != NULL );
860 return ( *__kmp_release_nested_user_lock_with_checks_ )( lck, gtid );
863 extern void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
865 static inline void __kmp_init_nested_user_lock_with_checks( kmp_user_lock_p lck )
867 KMP_DEBUG_ASSERT( __kmp_init_nested_user_lock_with_checks_ != NULL );
868 ( *__kmp_init_nested_user_lock_with_checks_ )( lck );
871 extern void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
874 __kmp_destroy_nested_user_lock_with_checks( kmp_user_lock_p lck )
876 KMP_DEBUG_ASSERT( __kmp_destroy_nested_user_lock_with_checks_ != NULL );
877 ( *__kmp_destroy_nested_user_lock_with_checks_ )( lck );
895 extern int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck );
899 extern const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck );
902 __kmp_get_user_lock_location( kmp_user_lock_p lck )
904 if ( __kmp_get_user_lock_location_ != NULL ) {
905 return ( *__kmp_get_user_lock_location_ )( lck );
912 extern void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc );
915 __kmp_set_user_lock_location( kmp_user_lock_p lck,
const ident_t *loc )
917 if ( __kmp_set_user_lock_location_ != NULL ) {
918 ( *__kmp_set_user_lock_location_ )( lck, loc );
922 extern kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck );
924 extern void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags );
927 __kmp_set_user_lock_flags( kmp_user_lock_p lck, kmp_lock_flags_t flags )
929 if ( __kmp_set_user_lock_flags_ != NULL ) {
930 ( *__kmp_set_user_lock_flags_ )( lck, flags );
937 extern void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind );
942 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) { \ 943 __kmp_acquire##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p, kmp_int32 ) ) \ 944 __kmp_acquire##nest##kind##_##suffix; \ 945 __kmp_release##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 946 __kmp_release##nest##kind##_##suffix; \ 947 __kmp_test##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 948 __kmp_test##nest##kind##_##suffix; \ 949 __kmp_init##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \ 950 __kmp_init##nest##kind##_##suffix; \ 951 __kmp_destroy##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \ 952 __kmp_destroy##nest##kind##_##suffix; \ 955 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock) 956 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks) 957 #define KMP_BIND_NESTED_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock) 958 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks) 986 struct kmp_lock_table {
987 kmp_lock_index_t used;
988 kmp_lock_index_t allocated;
989 kmp_user_lock_p * table;
992 typedef struct kmp_lock_table kmp_lock_table_t;
994 extern kmp_lock_table_t __kmp_user_lock_table;
995 extern kmp_user_lock_p __kmp_lock_pool;
997 struct kmp_block_of_locks {
998 struct kmp_block_of_locks * next_block;
1002 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
1004 extern kmp_block_of_locks_t *__kmp_lock_blocks;
1005 extern int __kmp_num_locks_in_block;
1007 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid, kmp_lock_flags_t flags );
1008 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck );
1009 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func );
1010 extern void __kmp_cleanup_user_locks();
1012 #define KMP_CHECK_USER_LOCK_INIT() \ 1014 if ( ! TCR_4( __kmp_init_user_locks ) ) { \ 1015 __kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); \ 1016 if ( ! TCR_4( __kmp_init_user_locks ) ) { \ 1017 TCW_4( __kmp_init_user_locks, TRUE ); \ 1019 __kmp_release_bootstrap_lock( &__kmp_initz_lock ); \ 1023 #endif // KMP_USE_DYNAMIC_LOCK 1028 #if KMP_USE_DYNAMIC_LOCK 1030 #define DYNA_HAS_FUTEX (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) 1031 #define DYNA_HAS_HLE (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC) 1032 #define DYNA_USE_FAST_FUTEX 0 && DYNA_HAS_FUTEX 1033 #define DYNA_USE_FAST_TAS 1 && DYNA_HAS_FUTEX 1040 # define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) 1041 # define DYNA_LAST_D_LOCK_SEQ lockseq_hle 1043 # define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) 1044 # define DYNA_LAST_D_LOCK_SEQ lockseq_futex 1045 # endif // DYNA_HAS_HLE 1046 # if KMP_USE_ADAPTIVE_LOCKS 1047 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \ 1048 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1049 m(nested_queuing, a) m(nested_drdpa, a) 1051 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \ 1052 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1053 m(nested_queuing, a) m(nested_drdpa, a) 1054 # endif // KMP_USE_ADAPTIVE_LOCKS 1057 # define FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) 1058 # define DYNA_LAST_D_LOCK_SEQ lockseq_hle 1060 # define FOREACH_D_LOCK(m, a) m(tas, a) 1061 # define DYNA_LAST_D_LOCK_SEQ lockseq_tas 1062 # endif // DYNA_HAS_HLE 1063 # if KMP_USE_ADAPTIVE_LOCKS 1064 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \ 1065 m(nested_tas, a) m(nested_ticket, a) \ 1066 m(nested_queuing, a) m(nested_drdpa, a) 1068 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \ 1069 m(nested_tas, a) m(nested_ticket, a) \ 1070 m(nested_queuing, a) m(nested_drdpa, a) 1071 # endif // KMP_USE_ADAPTIVE_LOCKS 1072 #endif // DYNA_HAS_FUTEX 1075 #define DYNA_LOCK_VALUE_SHIFT 8 1076 #define DYNA_LOCK_TYPE_MASK ((1<<DYNA_LOCK_VALUE_SHIFT)-1) 1077 #define DYNA_NUM_D_LOCKS DYNA_LAST_D_LOCK_SEQ 1078 #define DYNA_NUM_I_LOCKS (locktag_nested_drdpa+1) 1081 typedef kmp_uint32 kmp_dyna_lock_t;
1086 lockseq_indirect = 0,
1087 #define expand_seq(l,a) lockseq_##l, 1088 FOREACH_D_LOCK(expand_seq, 0)
1089 FOREACH_I_LOCK(expand_seq, 0)
1091 } kmp_dyna_lockseq_t;
1095 #define expand_tag(l,a) locktag_##l, 1096 FOREACH_I_LOCK(expand_tag, 0)
1098 } kmp_indirect_locktag_t;
1101 #define DYNA_IS_D_LOCK(seq) (seq >= lockseq_tas && seq <= DYNA_LAST_D_LOCK_SEQ) 1102 #define DYNA_IS_I_LOCK(seq) (seq >= lockseq_ticket && seq <= lockseq_nested_drdpa) 1103 #define DYNA_GET_I_TAG(seq) (kmp_indirect_locktag_t)(seq - lockseq_ticket) 1104 #define DYNA_GET_D_TAG(seq) (seq<<1 | 1) 1108 #define expand_tag(l,a) locktag_##l = DYNA_GET_D_TAG(lockseq_##l), 1109 FOREACH_D_LOCK(expand_tag, 0)
1111 } kmp_direct_locktag_t;
1115 kmp_user_lock_p lock;
1116 kmp_indirect_locktag_t type;
1117 } kmp_indirect_lock_t;
1120 extern void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1121 extern void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *);
1122 extern void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32);
1123 extern void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32);
1124 extern int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32);
1127 extern void (*__kmp_indirect_init_ops[])(kmp_user_lock_p);
1128 extern void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p);
1129 extern void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32);
1130 extern void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32);
1131 extern int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32);
1134 #define DYNA_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & DYNA_LOCK_TYPE_MASK & -(*((kmp_dyna_lock_t *)(l)) & 1)) 1137 #define DYNA_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1) 1140 #define DYNA_D_LOCK_FUNC(l, op) __kmp_direct_##op##_ops[DYNA_EXTRACT_D_TAG(l)] 1143 #define DYNA_I_LOCK_FUNC(l, op) __kmp_indirect_##op##_ops[((kmp_indirect_lock_t *)(l))->type] 1146 #define DYNA_INIT_D_LOCK(l, seq) __kmp_direct_init_ops[DYNA_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq) 1149 #define DYNA_INIT_I_LOCK(l, seq) __kmp_direct_init_ops[0]((kmp_dyna_lock_t *)(l), seq) 1152 #define DYNA_LOCK_FREE(type) (locktag_##type) 1155 #define DYNA_LOCK_BUSY(v, type) ((v)<<DYNA_LOCK_VALUE_SHIFT | locktag_##type) 1158 #define DYNA_LOCK_STRIP(v) ((v)>>DYNA_LOCK_VALUE_SHIFT) 1161 #define DYNA_STORE_LOCK_SEQ(type) (__kmp_user_lock_seq = lockseq_##type) 1164 extern void __kmp_init_lock_hinted(
void **,
int);
1165 extern void __kmp_init_nest_lock_hinted(
void **,
int);
1168 extern void __kmp_init_dynamic_user_locks();
1171 extern kmp_indirect_lock_t * __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1174 extern void __kmp_cleanup_indirect_user_locks();
1177 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1180 extern void (*__kmp_indirect_set_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *);
1181 #define DYNA_SET_I_LOCK_LOCATION(lck, loc) { \ 1182 if (__kmp_indirect_set_location[(lck)->type] != NULL) \ 1183 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \ 1187 extern void (*__kmp_indirect_set_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t);
1188 #define DYNA_SET_I_LOCK_FLAGS(lck, flag) { \ 1189 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \ 1190 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \ 1194 extern const ident_t * (*__kmp_indirect_get_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p);
1195 #define DYNA_GET_I_LOCK_LOCATION(lck) ( __kmp_indirect_get_location[(lck)->type] != NULL \ 1196 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \ 1200 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p);
1201 #define DYNA_GET_I_LOCK_FLAGS(lck) ( __kmp_indirect_get_flags[(lck)->type] != NULL \ 1202 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \ 1209 extern kmp_indirect_lock_t **__kmp_indirect_lock_table;
1211 extern kmp_lock_index_t __kmp_indirect_lock_table_size;
1213 extern kmp_lock_index_t __kmp_indirect_lock_table_next;
1217 extern int __kmp_num_locks_in_block;
1220 #define DYNA_LOOKUP_I_LOCK(l) ( (OMP_LOCK_T_SIZE < sizeof(void *)) \ 1221 ? __kmp_indirect_lock_table[DYNA_EXTRACT_I_INDEX(l)] \ 1222 : *((kmp_indirect_lock_t **)l) ) 1226 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1228 #else // KMP_USE_DYNAMIC_LOCK 1230 # define DYNA_LOCK_BUSY(v, type) (v) 1231 # define DYNA_LOCK_FREE(type) 0 1232 # define DYNA_LOCK_STRIP(v) (v) 1233 # define DYNA_STORE_LOCK_SEQ(seq) 1235 #endif // KMP_USE_DYNAMIC_LOCK 1239 #endif // __cplusplus