17 #include "kmp_wait_release.h" 18 #include "kmp_stats.h" 22 #include <immintrin.h> 23 #define USE_NGO_STORES 1 26 #if KMP_MIC && USE_NGO_STORES 28 #define ngo_load(src) __m512d Vt = _mm512_load_pd((void *)(src)) 29 #define ngo_store_icvs(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt) 30 #define ngo_store_go(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt) 31 #define ngo_sync() __asm__ volatile ("lock; addl $0,0(%%rsp)" ::: "memory") 33 #define ngo_load(src) ((void)0) 34 #define ngo_store_icvs(dst, src) copy_icvs((dst), (src)) 35 #define ngo_store_go(dst, src) KMP_MEMCPY((dst), (src), CACHE_LINE) 36 #define ngo_sync() ((void)0) 39 void __kmp_print_structure(
void);
45 __kmp_linear_barrier_gather(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
46 void (*reduce)(
void *,
void *)
47 USE_ITT_BUILD_ARG(
void * itt_sync_obj) )
50 register kmp_team_t *team = this_thr->th.th_team;
51 register kmp_bstate_t *thr_bar = & this_thr->th.th_bar[bt].bb;
52 register kmp_info_t **other_threads = team->t.t_threads;
54 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
55 gtid, team->t.t_id, tid, bt));
56 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
58 #if USE_ITT_BUILD && USE_ITT_NOTIFY 60 if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
61 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
65 if (!KMP_MASTER_TID(tid)) {
66 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)" 67 "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
68 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, &thr_bar->b_arrived,
69 thr_bar->b_arrived, thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
73 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]);
76 register kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
77 register int nproc = this_thr->th.th_team_nproc;
80 register kmp_uint new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP;
83 for (i=1; i<nproc; ++i) {
87 KMP_CACHE_PREFETCH(&other_threads[i+1]->th.th_bar[bt].bb.b_arrived);
89 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) " 90 "arrived(%p) == %u\n", gtid, team->t.t_id, tid,
91 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
92 &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state));
95 kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state);
96 flag.wait(this_thr, FALSE
97 USE_ITT_BUILD_ARG(itt_sync_obj) );
98 #if USE_ITT_BUILD && USE_ITT_NOTIFY 100 if (__kmp_forkjoin_frames_mode == 2) {
101 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
102 other_threads[i]->th.th_bar_min_time);
106 KA_TRACE(100, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid,
107 team->t.t_id, tid, __kmp_gtid_from_tid(i, team), team->t.t_id, i));
108 (*reduce)(this_thr->th.th_local.reduce_data,
109 other_threads[i]->th.th_local.reduce_data);
113 team_bar->b_arrived = new_state;
114 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
115 gtid, team->t.t_id, tid, team->t.t_id, &team_bar->b_arrived, new_state));
117 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
118 gtid, team->t.t_id, tid, bt));
122 __kmp_linear_barrier_release(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
124 USE_ITT_BUILD_ARG(
void *itt_sync_obj) )
127 register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
128 register kmp_team_t *team;
130 if (KMP_MASTER_TID(tid)) {
131 register unsigned int i;
132 register kmp_uint32 nproc = this_thr->th.th_team_nproc;
133 register kmp_info_t **other_threads;
135 team = __kmp_threads[gtid]->th.th_team;
136 KMP_DEBUG_ASSERT(team != NULL);
137 other_threads = team->t.t_threads;
139 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d(%d:%d) master enter for barrier type %d\n",
140 gtid, team->t.t_id, tid, bt));
143 #if KMP_BARRIER_ICV_PUSH 145 if (propagate_icvs) {
146 ngo_load(&team->t.t_implicit_task_taskdata[0].td_icvs);
147 for (i=1; i<nproc; ++i) {
148 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[i], team, i, FALSE);
149 ngo_store_icvs(&team->t.t_implicit_task_taskdata[i].td_icvs,
150 &team->t.t_implicit_task_taskdata[0].td_icvs);
155 #endif // KMP_BARRIER_ICV_PUSH 158 for (i=1; i<nproc; ++i) {
162 KMP_CACHE_PREFETCH(&other_threads[i+1]->th.th_bar[bt].bb.b_go);
164 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d) " 165 "go(%p): %u => %u\n", gtid, team->t.t_id, tid,
166 other_threads[i]->th.th_info.ds.ds_gtid, team->t.t_id, i,
167 &other_threads[i]->th.th_bar[bt].bb.b_go,
168 other_threads[i]->th.th_bar[bt].bb.b_go,
169 other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP));
170 kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_go, other_threads[i]);
175 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
176 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
177 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
178 flag.wait(this_thr, TRUE
179 USE_ITT_BUILD_ARG(itt_sync_obj) );
180 #if USE_ITT_BUILD && USE_ITT_NOTIFY 181 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
183 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
185 __kmp_itt_task_starting(itt_sync_obj);
187 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
190 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
191 if (itt_sync_obj != NULL)
193 __kmp_itt_task_finished(itt_sync_obj);
197 if ( bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done) )
201 tid = __kmp_tid_from_gtid(gtid);
202 team = __kmp_threads[gtid]->th.th_team;
204 KMP_DEBUG_ASSERT(team != NULL);
205 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
206 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
207 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
210 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
211 gtid, team->t.t_id, tid, bt));
216 __kmp_tree_barrier_gather(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
217 void (*reduce)(
void *,
void *)
218 USE_ITT_BUILD_ARG(
void *itt_sync_obj) )
221 register kmp_team_t *team = this_thr->th.th_team;
222 register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
223 register kmp_info_t **other_threads = team->t.t_threads;
224 register kmp_uint32 nproc = this_thr->th.th_team_nproc;
225 register kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
226 register kmp_uint32 branch_factor = 1 << branch_bits;
227 register kmp_uint32 child;
228 register kmp_uint32 child_tid;
229 register kmp_uint new_state;
231 KA_TRACE(20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
232 gtid, team->t.t_id, tid, bt));
233 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
235 #if USE_ITT_BUILD && USE_ITT_NOTIFY 237 if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
238 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
242 child_tid = (tid << branch_bits) + 1;
243 if (child_tid < nproc) {
245 new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
248 register kmp_info_t *child_thr = other_threads[child_tid];
249 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
252 if (child+1 <= branch_factor && child_tid+1 < nproc)
253 KMP_CACHE_PREFETCH(&other_threads[child_tid+1]->th.th_bar[bt].bb.b_arrived);
255 KA_TRACE(20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) " 256 "arrived(%p) == %u\n", gtid, team->t.t_id, tid,
257 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid,
258 &child_bar->b_arrived, new_state));
260 kmp_flag_64 flag(&child_bar->b_arrived, new_state);
261 flag.wait(this_thr, FALSE
262 USE_ITT_BUILD_ARG(itt_sync_obj) );
263 #if USE_ITT_BUILD && USE_ITT_NOTIFY 265 if (__kmp_forkjoin_frames_mode == 2) {
266 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
267 child_thr->th.th_bar_min_time);
271 KA_TRACE(100, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
272 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
273 team->t.t_id, child_tid));
274 (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
279 while (child <= branch_factor && child_tid < nproc);
282 if (!KMP_MASTER_TID(tid)) {
283 register kmp_int32 parent_tid = (tid - 1) >> branch_bits;
285 KA_TRACE(20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) " 286 "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
287 __kmp_gtid_from_tid(parent_tid, team), team->t.t_id, parent_tid,
288 &thr_bar->b_arrived, thr_bar->b_arrived,
289 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
294 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]);
299 team->t.t_bar[bt].b_arrived = new_state;
301 team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
302 KA_TRACE(20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
303 gtid, team->t.t_id, tid, team->t.t_id,
304 &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
306 KA_TRACE(20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
307 gtid, team->t.t_id, tid, bt));
311 __kmp_tree_barrier_release(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
313 USE_ITT_BUILD_ARG(
void *itt_sync_obj) )
316 register kmp_team_t *team;
317 register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
318 register kmp_uint32 nproc;
319 register kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[bt];
320 register kmp_uint32 branch_factor = 1 << branch_bits;
321 register kmp_uint32 child;
322 register kmp_uint32 child_tid;
325 if (!KMP_MASTER_TID(tid)) {
326 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d wait go(%p) == %u\n",
327 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
329 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
330 flag.wait(this_thr, TRUE
331 USE_ITT_BUILD_ARG(itt_sync_obj) );
332 #if USE_ITT_BUILD && USE_ITT_NOTIFY 333 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
335 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
337 __kmp_itt_task_starting(itt_sync_obj);
339 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
342 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
343 if (itt_sync_obj != NULL)
345 __kmp_itt_task_finished(itt_sync_obj);
349 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
353 team = __kmp_threads[gtid]->th.th_team;
354 KMP_DEBUG_ASSERT(team != NULL);
355 tid = __kmp_tid_from_gtid(gtid);
357 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
358 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
359 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
362 team = __kmp_threads[gtid]->th.th_team;
363 KMP_DEBUG_ASSERT(team != NULL);
364 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d(%d:%d) master enter for barrier type %d\n",
365 gtid, team->t.t_id, tid, bt));
367 nproc = this_thr->th.th_team_nproc;
368 child_tid = (tid << branch_bits) + 1;
370 if (child_tid < nproc) {
371 register kmp_info_t **other_threads = team->t.t_threads;
375 register kmp_info_t *child_thr = other_threads[child_tid];
376 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
379 if (child+1 <= branch_factor && child_tid+1 < nproc)
380 KMP_CACHE_PREFETCH(&other_threads[child_tid+1]->th.th_bar[bt].bb.b_go);
383 #if KMP_BARRIER_ICV_PUSH 385 if (propagate_icvs) {
386 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[child_tid],
387 team, child_tid, FALSE);
388 copy_icvs(&team->t.t_implicit_task_taskdata[child_tid].td_icvs,
389 &team->t.t_implicit_task_taskdata[0].td_icvs);
392 #endif // KMP_BARRIER_ICV_PUSH 393 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)" 394 "go(%p): %u => %u\n", gtid, team->t.t_id, tid,
395 __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
396 child_tid, &child_bar->b_go, child_bar->b_go,
397 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
399 kmp_flag_64 flag(&child_bar->b_go, child_thr);
404 while (child <= branch_factor && child_tid < nproc);
406 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
407 gtid, team->t.t_id, tid, bt));
413 __kmp_hyper_barrier_gather(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
414 void (*reduce)(
void *,
void *)
415 USE_ITT_BUILD_ARG(
void *itt_sync_obj) )
418 register kmp_team_t *team = this_thr->th.th_team;
419 register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
420 register kmp_info_t **other_threads = team->t.t_threads;
421 register kmp_uint new_state = KMP_BARRIER_UNUSED_STATE;
422 register kmp_uint32 num_threads = this_thr->th.th_team_nproc;
423 register kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
424 register kmp_uint32 branch_factor = 1 << branch_bits;
425 register kmp_uint32 offset;
426 register kmp_uint32 level;
428 KA_TRACE(20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
429 gtid, team->t.t_id, tid, bt));
431 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
433 #if USE_ITT_BUILD && USE_ITT_NOTIFY 435 if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
436 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
441 kmp_flag_64 p_flag(&thr_bar->b_arrived);
442 for (level=0, offset=1; offset<num_threads; level+=branch_bits, offset<<=branch_bits)
444 register kmp_uint32 child;
445 register kmp_uint32 child_tid;
447 if (((tid >> level) & (branch_factor - 1)) != 0) {
448 register kmp_int32 parent_tid = tid & ~((1 << (level + branch_bits)) -1);
450 KA_TRACE(20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) " 451 "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
452 __kmp_gtid_from_tid(parent_tid, team), team->t.t_id, parent_tid,
453 &thr_bar->b_arrived, thr_bar->b_arrived,
454 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
459 p_flag.set_waiter(other_threads[parent_tid]);
465 if (new_state == KMP_BARRIER_UNUSED_STATE)
466 new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
467 for (child=1, child_tid=tid+(1 << level); child<branch_factor && child_tid<num_threads;
468 child++, child_tid+=(1 << level))
470 register kmp_info_t *child_thr = other_threads[child_tid];
471 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
473 register kmp_uint32 next_child_tid = child_tid + (1 << level);
475 if (child+1 < branch_factor && next_child_tid < num_threads)
476 KMP_CACHE_PREFETCH(&other_threads[next_child_tid]->th.th_bar[bt].bb.b_arrived);
478 KA_TRACE(20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) " 479 "arrived(%p) == %u\n", gtid, team->t.t_id, tid,
480 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid,
481 &child_bar->b_arrived, new_state));
483 kmp_flag_64 c_flag(&child_bar->b_arrived, new_state);
484 c_flag.wait(this_thr, FALSE
485 USE_ITT_BUILD_ARG(itt_sync_obj) );
486 #if USE_ITT_BUILD && USE_ITT_NOTIFY 488 if (__kmp_forkjoin_frames_mode == 2) {
489 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
490 child_thr->th.th_bar_min_time);
494 KA_TRACE(100, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
495 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
496 team->t.t_id, child_tid));
497 (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
502 if (KMP_MASTER_TID(tid)) {
504 if (new_state == KMP_BARRIER_UNUSED_STATE)
505 team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
507 team->t.t_bar[bt].b_arrived = new_state;
508 KA_TRACE(20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
509 gtid, team->t.t_id, tid, team->t.t_id,
510 &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
512 KA_TRACE(20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
513 gtid, team->t.t_id, tid, bt));
517 #define KMP_REVERSE_HYPER_BAR 519 __kmp_hyper_barrier_release(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
521 USE_ITT_BUILD_ARG(
void *itt_sync_obj) )
524 register kmp_team_t *team;
525 register kmp_bstate_t *thr_bar = & this_thr -> th.th_bar[ bt ].bb;
526 register kmp_info_t **other_threads;
527 register kmp_uint32 num_threads;
528 register kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[ bt ];
529 register kmp_uint32 branch_factor = 1 << branch_bits;
530 register kmp_uint32 child;
531 register kmp_uint32 child_tid;
532 register kmp_uint32 offset;
533 register kmp_uint32 level;
538 if (KMP_MASTER_TID(tid)) {
539 team = __kmp_threads[gtid]->th.th_team;
540 KMP_DEBUG_ASSERT(team != NULL);
541 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d(%d:%d) master enter for barrier type %d\n",
542 gtid, team->t.t_id, tid, bt));
543 #if KMP_BARRIER_ICV_PUSH 544 if (propagate_icvs) {
545 copy_icvs(&thr_bar->th_fixed_icvs, &team->t.t_implicit_task_taskdata[tid].td_icvs);
550 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n",
551 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
553 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
554 flag.wait(this_thr, TRUE
555 USE_ITT_BUILD_ARG(itt_sync_obj) );
556 #if USE_ITT_BUILD && USE_ITT_NOTIFY 557 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
559 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
561 __kmp_itt_task_starting(itt_sync_obj);
563 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
566 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
567 if (itt_sync_obj != NULL)
569 __kmp_itt_task_finished(itt_sync_obj);
573 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
577 team = __kmp_threads[gtid]->th.th_team;
578 KMP_DEBUG_ASSERT(team != NULL);
579 tid = __kmp_tid_from_gtid(gtid);
581 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
582 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
583 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
586 num_threads = this_thr->th.th_team_nproc;
587 other_threads = team->t.t_threads;
589 #ifdef KMP_REVERSE_HYPER_BAR 591 for (level=0, offset=1; offset<num_threads && (((tid>>level) & (branch_factor-1)) == 0);
592 level+=branch_bits, offset<<=branch_bits);
595 for (level-=branch_bits, offset>>=branch_bits; offset != 0;
596 level-=branch_bits, offset>>=branch_bits)
599 for (level=0, offset=1; offset<num_threads; level+=branch_bits, offset<<=branch_bits)
602 #ifdef KMP_REVERSE_HYPER_BAR 605 child = num_threads >> ((level==0)?level:level-1);
606 for (child=(child<branch_factor-1) ? child : branch_factor-1, child_tid=tid+(child<<level);
607 child>=1; child--, child_tid-=(1<<level))
609 if (((tid >> level) & (branch_factor - 1)) != 0)
613 for (child=1, child_tid=tid+(1<<level); child<branch_factor && child_tid<num_threads;
614 child++, child_tid+=(1<<level))
615 #endif // KMP_REVERSE_HYPER_BAR 617 if (child_tid >= num_threads)
continue;
619 register kmp_info_t *child_thr = other_threads[child_tid];
620 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
622 register kmp_uint32 next_child_tid = child_tid - (1 << level);
624 # ifdef KMP_REVERSE_HYPER_BAR 625 if (child-1 >= 1 && next_child_tid < num_threads)
627 if (child+1 < branch_factor && next_child_tid < num_threads)
628 # endif // KMP_REVERSE_HYPER_BAR 629 KMP_CACHE_PREFETCH(&other_threads[next_child_tid]->th.th_bar[bt].bb.b_go);
632 #if KMP_BARRIER_ICV_PUSH 634 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
635 #endif // KMP_BARRIER_ICV_PUSH 637 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)" 638 "go(%p): %u => %u\n", gtid, team->t.t_id, tid,
639 __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
640 child_tid, &child_bar->b_go, child_bar->b_go,
641 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
643 kmp_flag_64 flag(&child_bar->b_go, child_thr);
648 #if KMP_BARRIER_ICV_PUSH 649 if (propagate_icvs && !KMP_MASTER_TID(tid)) {
650 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid, FALSE);
651 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, &thr_bar->th_fixed_icvs);
654 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
655 gtid, team->t.t_id, tid, bt));
667 __kmp_init_hierarchical_barrier_thread(
enum barrier_type bt, kmp_bstate_t *thr_bar, kmp_uint32 nproc,
668 int gtid,
int tid, kmp_team_t *team)
671 bool uninitialized = thr_bar->team == NULL;
672 bool team_changed = team != thr_bar->team;
673 bool team_sz_changed = nproc != thr_bar->nproc;
674 bool tid_changed = tid != thr_bar->old_tid;
677 if (uninitialized || team_sz_changed) {
678 __kmp_get_hierarchy(nproc, thr_bar);
681 if (uninitialized || team_sz_changed || tid_changed) {
682 thr_bar->my_level = thr_bar->depth-1;
683 thr_bar->parent_tid = -1;
684 if (!KMP_MASTER_TID(tid)) {
686 while (d<thr_bar->depth) {
688 if (d == thr_bar->depth-2) {
689 thr_bar->parent_tid = 0;
690 thr_bar->my_level = d;
693 else if ((rem = tid%thr_bar->skip_per_level[d+1]) != 0) {
695 thr_bar->parent_tid = tid - rem;
696 thr_bar->my_level = d;
702 thr_bar->offset = 7-(tid-thr_bar->parent_tid-1);
703 thr_bar->old_tid = tid;
704 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
706 if (uninitialized || team_changed || tid_changed) {
707 thr_bar->team = team;
708 thr_bar->parent_bar = &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
711 if (uninitialized || team_sz_changed || tid_changed) {
712 thr_bar->nproc = nproc;
713 thr_bar->leaf_kids = thr_bar->base_leaf_kids;
714 if (thr_bar->my_level == 0) thr_bar->leaf_kids=0;
715 if (thr_bar->leaf_kids && (kmp_uint32)tid+thr_bar->leaf_kids+1 > nproc)
716 thr_bar->leaf_kids = nproc - tid - 1;
717 thr_bar->leaf_state = 0;
718 for (
int i=0; i<thr_bar->leaf_kids; ++i) ((
char *)&(thr_bar->leaf_state))[7-i] = 1;
724 __kmp_hierarchical_barrier_gather(
enum barrier_type bt, kmp_info_t *this_thr,
725 int gtid,
int tid,
void (*reduce) (
void *,
void *)
726 USE_ITT_BUILD_ARG(
void * itt_sync_obj) )
729 register kmp_team_t *team = this_thr->th.th_team;
730 register kmp_bstate_t *thr_bar = & this_thr->th.th_bar[bt].bb;
731 register kmp_uint32 nproc = this_thr->th.th_team_nproc;
732 register kmp_info_t **other_threads = team->t.t_threads;
733 register kmp_uint64 new_state;
735 int level = team->t.t_level;
736 if (other_threads[0]->th.th_teams_microtask)
737 if (this_thr->th.th_teams_size.nteams > 1)
739 if (level == 1) thr_bar->use_oncore_barrier = 1;
740 else thr_bar->use_oncore_barrier = 0;
742 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
743 gtid, team->t.t_id, tid, bt));
744 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
746 #if USE_ITT_BUILD && USE_ITT_NOTIFY 748 if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
749 this_thr->th.th_bar_arrive_time = __itt_get_timestamp();
753 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid, team);
755 if (thr_bar->my_level) {
756 register kmp_int32 child_tid;
757 new_state = (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
758 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && thr_bar->use_oncore_barrier) {
759 if (thr_bar->leaf_kids) {
760 kmp_uint64 leaf_state = KMP_MASTER_TID(tid) ? thr_bar->b_arrived | thr_bar->leaf_state : (kmp_uint64)team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
761 kmp_flag_64 flag(&thr_bar->b_arrived, leaf_state);
762 flag.wait(this_thr, FALSE
763 USE_ITT_BUILD_ARG(itt_sync_obj) );
765 for (child_tid=tid+1; child_tid<=tid+thr_bar->leaf_kids; ++child_tid) {
766 KA_TRACE(100, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
767 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
768 team->t.t_id, child_tid));
769 (*reduce)(this_thr->th.th_local.reduce_data, other_threads[child_tid]->th.th_local.reduce_data);
772 (void) KMP_TEST_THEN_AND64((
volatile kmp_int64 *)&thr_bar->b_arrived, ~(thr_bar->leaf_state));
775 for (kmp_uint32 d=1; d<thr_bar->my_level; ++d) {
776 kmp_uint32 last = tid+thr_bar->skip_per_level[d+1], skip = thr_bar->skip_per_level[d];
777 if (last > nproc) last = nproc;
778 for (child_tid=tid+skip; child_tid<(int)last; child_tid+=skip) {
779 register kmp_info_t *child_thr = other_threads[child_tid];
780 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
781 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) " 782 "arrived(%p) == %u\n",
783 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
784 team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
785 kmp_flag_64 flag(&child_bar->b_arrived, new_state);
786 flag.wait(this_thr, FALSE
787 USE_ITT_BUILD_ARG(itt_sync_obj) );
789 KA_TRACE(100, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
790 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
791 team->t.t_id, child_tid));
792 (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
798 for (kmp_uint32 d=0; d<thr_bar->my_level; ++d) {
799 kmp_uint32 last = tid+thr_bar->skip_per_level[d+1], skip = thr_bar->skip_per_level[d];
800 if (last > nproc) last = nproc;
801 for (child_tid=tid+skip; child_tid<(int)last; child_tid+=skip) {
802 register kmp_info_t *child_thr = other_threads[child_tid];
803 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
804 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) " 805 "arrived(%p) == %u\n",
806 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
807 team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
808 kmp_flag_64 flag(&child_bar->b_arrived, new_state);
809 flag.wait(this_thr, FALSE
810 USE_ITT_BUILD_ARG(itt_sync_obj) );
812 KA_TRACE(100, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
813 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
814 team->t.t_id, child_tid));
815 (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
823 if (!KMP_MASTER_TID(tid)) {
824 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) " 825 "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
826 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id, thr_bar->parent_tid,
827 &thr_bar->b_arrived, thr_bar->b_arrived, thr_bar->b_arrived+KMP_BARRIER_STATE_BUMP));
830 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME
831 || !thr_bar->use_oncore_barrier) {
832 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]);
836 thr_bar->b_arrived = (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
837 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived, thr_bar->offset);
838 flag.set_waiter(other_threads[thr_bar->parent_tid]);
842 team->t.t_bar[bt].b_arrived = (kmp_uint32)new_state;
843 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
844 gtid, team->t.t_id, tid, team->t.t_id, &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
847 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
848 gtid, team->t.t_id, tid, bt));
852 __kmp_hierarchical_barrier_release(
enum barrier_type bt, kmp_info_t *this_thr,
int gtid,
int tid,
854 USE_ITT_BUILD_ARG(
void * itt_sync_obj) )
857 register kmp_team_t *team;
858 register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
859 register kmp_uint32 nproc;
860 bool team_change =
false;
862 if (KMP_MASTER_TID(tid)) {
863 team = __kmp_threads[gtid]->th.th_team;
864 KMP_DEBUG_ASSERT(team != NULL);
865 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) master entered barrier type %d\n",
866 gtid, team->t.t_id, tid, bt));
870 if (!thr_bar->use_oncore_barrier || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME
871 || thr_bar->my_level != 0 || thr_bar->team == NULL) {
873 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
874 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
875 flag.wait(this_thr, TRUE
876 USE_ITT_BUILD_ARG(itt_sync_obj) );
877 TCW_8(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
881 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
882 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP, thr_bar->offset,
884 USE_ITT_BUILD_ARG(itt_sync_obj) );
885 flag.wait(this_thr, TRUE
886 USE_ITT_BUILD_ARG(itt_sync_obj) );
887 if (thr_bar->wait_flag == KMP_BARRIER_SWITCHING) {
888 TCW_8(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
891 ((
char*)&(thr_bar->parent_bar->b_go))[thr_bar->offset] = 0;
894 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
896 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
899 team = __kmp_threads[gtid]->th.th_team;
900 KMP_DEBUG_ASSERT(team != NULL);
901 tid = __kmp_tid_from_gtid(gtid);
903 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
904 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
908 int level = team->t.t_level;
909 if (team->t.t_threads[0]->th.th_teams_microtask ) {
910 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && this_thr->th.th_teams_level == level)
912 if( this_thr->th.th_teams_size.nteams > 1 )
915 if (level == 1) thr_bar->use_oncore_barrier = 1;
916 else thr_bar->use_oncore_barrier = 0;
917 nproc = this_thr->th.th_team_nproc;
920 unsigned short int old_leaf_kids = thr_bar->leaf_kids;
921 kmp_uint64 old_leaf_state = thr_bar->leaf_state;
922 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid, team);
924 if (team_change) old_leaf_kids = 0;
926 #if KMP_BARRIER_ICV_PUSH 927 if (propagate_icvs) {
928 if (KMP_MASTER_TID(tid)) {
929 copy_icvs(&thr_bar->th_fixed_icvs, &team->t.t_implicit_task_taskdata[tid].td_icvs);
931 else if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && thr_bar->use_oncore_barrier) {
932 if (!thr_bar->my_level)
934 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
935 &thr_bar->parent_bar->th_fixed_icvs);
939 if (thr_bar->my_level)
940 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
942 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
943 &thr_bar->parent_bar->th_fixed_icvs);
946 #endif // KMP_BARRIER_ICV_PUSH 949 if (thr_bar->my_level) {
950 register kmp_int32 child_tid;
952 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && thr_bar->use_oncore_barrier) {
953 if (KMP_MASTER_TID(tid)) {
955 thr_bar->b_go = KMP_BARRIER_STATE_BUMP;
957 ngo_load(&thr_bar->th_fixed_icvs);
959 for (child_tid=thr_bar->skip_per_level[1]; child_tid<(
int)nproc; child_tid+=thr_bar->skip_per_level[1]) {
960 register kmp_bstate_t *child_bar = &team->t.t_threads[child_tid]->th.th_bar[bt].bb;
961 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d)" 962 " go(%p): %u => %u\n",
963 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
964 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
965 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
967 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
971 TCW_8(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
973 if (thr_bar->leaf_kids) {
975 if (team_change || old_leaf_kids < thr_bar->leaf_kids) {
977 thr_bar->b_go |= old_leaf_state;
980 last = tid+thr_bar->skip_per_level[1];
981 if (last > nproc) last = nproc;
982 for (child_tid=tid+1+old_leaf_kids; child_tid<(int)last; ++child_tid) {
983 register kmp_info_t *child_thr = team->t.t_threads[child_tid];
984 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
985 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing" 986 " T#%d(%d:%d) go(%p): %u => %u\n",
987 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
988 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
989 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
991 kmp_flag_64 flag(&child_bar->b_go, child_thr);
996 thr_bar->b_go |= thr_bar->leaf_state;
1001 for (
int d=thr_bar->my_level-1; d>=0; --d) {
1002 last = tid+thr_bar->skip_per_level[d+1];
1003 kmp_uint32 skip = thr_bar->skip_per_level[d];
1004 if (last > nproc) last = nproc;
1005 for (child_tid=tid+skip; child_tid<(int)last; child_tid+=skip) {
1006 register kmp_info_t *child_thr = team->t.t_threads[child_tid];
1007 register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1008 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d)" 1009 " go(%p): %u => %u\n",
1010 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
1011 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
1012 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
1014 kmp_flag_64 flag(&child_bar->b_go, child_thr);
1019 #if KMP_BARRIER_ICV_PUSH 1020 if (propagate_icvs && !KMP_MASTER_TID(tid))
1021 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, &thr_bar->th_fixed_icvs);
1022 #endif // KMP_BARRIER_ICV_PUSH 1024 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
1025 gtid, team->t.t_id, tid, bt));
1035 __kmp_barrier(
enum barrier_type bt,
int gtid,
int is_split,
size_t reduce_size,
1036 void *reduce_data,
void (*reduce)(
void *,
void *))
1039 register int tid = __kmp_tid_from_gtid(gtid);
1040 register kmp_info_t *this_thr = __kmp_threads[gtid];
1041 register kmp_team_t *team = this_thr->th.th_team;
1042 register int status = 0;
1043 ident_t *loc = __kmp_threads[gtid]->th.th_ident;
1045 ompt_task_id_t my_task_id;
1046 ompt_parallel_id_t my_parallel_id;
1049 KA_TRACE(15, (
"__kmp_barrier: T#%d(%d:%d) has arrived\n",
1050 gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
1053 if (ompt_status & ompt_status_track) {
1055 if (ompt_status == ompt_status_track_callback) {
1056 my_task_id = team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id;
1057 my_parallel_id = team->t.ompt_team_info.parallel_id;
1060 if (this_thr->th.ompt_thread_info.state == ompt_state_wait_single) {
1061 if (ompt_callbacks.ompt_callback(ompt_event_single_others_end)) {
1062 ompt_callbacks.ompt_callback(ompt_event_single_others_end)(
1063 my_parallel_id, my_task_id);
1067 if (ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) {
1068 ompt_callbacks.ompt_callback(ompt_event_barrier_begin)(
1069 my_parallel_id, my_task_id);
1076 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
1080 if (! team->t.t_serialized) {
1083 void *itt_sync_obj = NULL;
1085 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1086 itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
1089 if (__kmp_tasking_mode == tskm_extra_barrier) {
1090 __kmp_tasking_barrier(team, this_thr, gtid);
1091 KA_TRACE(15, (
"__kmp_barrier: T#%d(%d:%d) past tasking barrier\n",
1092 gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
1098 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
1099 this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
1100 this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
1104 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1105 __kmp_itt_barrier_starting(gtid, itt_sync_obj);
1109 if (KMP_MASTER_TID(tid)) {
1110 team->t.t_bar[bt].b_master_arrived += 1;
1112 this_thr->th.th_bar[bt].bb.b_worker_arrived += 1;
1115 if (reduce != NULL) {
1117 this_thr->th.th_local.reduce_data = reduce_data;
1119 switch (__kmp_barrier_gather_pattern[bt]) {
1120 case bp_hyper_bar: {
1121 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]);
1122 __kmp_hyper_barrier_gather(bt, this_thr, gtid, tid, reduce
1123 USE_ITT_BUILD_ARG(itt_sync_obj) );
1126 case bp_hierarchical_bar: {
1127 __kmp_hierarchical_barrier_gather(bt, this_thr, gtid, tid, reduce
1128 USE_ITT_BUILD_ARG(itt_sync_obj));
1132 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]);
1133 __kmp_tree_barrier_gather(bt, this_thr, gtid, tid, reduce
1134 USE_ITT_BUILD_ARG(itt_sync_obj) );
1138 __kmp_linear_barrier_gather(bt, this_thr, gtid, tid, reduce
1139 USE_ITT_BUILD_ARG(itt_sync_obj) );
1145 if (KMP_MASTER_TID(tid)) {
1147 if (__kmp_tasking_mode != tskm_immediate_exec) {
1148 __kmp_task_team_wait(this_thr, team
1149 USE_ITT_BUILD_ARG(itt_sync_obj) );
1150 __kmp_task_team_setup(this_thr, team, 0, 0);
1154 team->t.t_bar[bt].b_team_arrived += 1;
1161 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1162 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1164 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1166 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && __kmp_forkjoin_frames_mode &&
1168 this_thr->th.th_teams_microtask == NULL &&
1170 team->t.t_active_level == 1)
1172 kmp_uint64 cur_time = __itt_get_timestamp();
1173 kmp_info_t **other_threads = team->t.t_threads;
1174 int nproc = this_thr->th.th_team_nproc;
1176 switch(__kmp_forkjoin_frames_mode) {
1178 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
1179 this_thr->th.th_frame_time = cur_time;
1182 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1, loc, nproc);
1185 if( __itt_metadata_add_ptr ) {
1187 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
1188 for (i=1; i<nproc; ++i) {
1189 delta += ( cur_time - other_threads[i]->th.th_bar_arrive_time );
1191 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, cur_time, delta, (kmp_uint64)( reduce != NULL));
1193 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
1194 this_thr->th.th_frame_time = cur_time;
1202 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1203 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1206 if (status == 1 || ! is_split) {
1207 switch (__kmp_barrier_release_pattern[bt]) {
1208 case bp_hyper_bar: {
1209 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
1210 __kmp_hyper_barrier_release(bt, this_thr, gtid, tid, FALSE
1211 USE_ITT_BUILD_ARG(itt_sync_obj) );
1214 case bp_hierarchical_bar: {
1215 __kmp_hierarchical_barrier_release(bt, this_thr, gtid, tid, FALSE
1216 USE_ITT_BUILD_ARG(itt_sync_obj) );
1220 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
1221 __kmp_tree_barrier_release(bt, this_thr, gtid, tid, FALSE
1222 USE_ITT_BUILD_ARG(itt_sync_obj) );
1226 __kmp_linear_barrier_release(bt, this_thr, gtid, tid, FALSE
1227 USE_ITT_BUILD_ARG(itt_sync_obj) );
1230 if (__kmp_tasking_mode != tskm_immediate_exec) {
1231 __kmp_task_team_sync(this_thr, team);
1239 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1240 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
1244 if (__kmp_tasking_mode != tskm_immediate_exec) {
1246 if ( this_thr->th.th_task_team != NULL ) {
1247 void *itt_sync_obj = NULL;
1249 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
1250 itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
1251 __kmp_itt_barrier_starting(gtid, itt_sync_obj);
1255 kmp_task_team_t * task_team;
1256 task_team = this_thr->th.th_task_team;
1257 KMP_DEBUG_ASSERT(task_team->tt.tt_found_proxy_tasks == TRUE);
1258 __kmp_task_team_wait(this_thr, team
1259 USE_ITT_BUILD_ARG(itt_sync_obj));
1260 __kmp_task_team_setup(this_thr, team, 0, 0);
1263 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1264 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
1269 KMP_DEBUG_ASSERT(team->t.t_task_team[this_thr->th.th_task_state] == NULL);
1270 KMP_DEBUG_ASSERT(this_thr->th.th_task_team == NULL);
1274 KA_TRACE(15, (
"__kmp_barrier: T#%d(%d:%d) is leaving with return value %d\n",
1275 gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid), status));
1278 if (ompt_status & ompt_status_track) {
1280 if ((ompt_status == ompt_status_track_callback) &&
1281 ompt_callbacks.ompt_callback(ompt_event_barrier_end)) {
1282 ompt_callbacks.ompt_callback(ompt_event_barrier_end)(
1283 my_parallel_id, my_task_id);
1286 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1295 __kmp_end_split_barrier(
enum barrier_type bt,
int gtid)
1298 int tid = __kmp_tid_from_gtid(gtid);
1299 kmp_info_t *this_thr = __kmp_threads[gtid];
1300 kmp_team_t *team = this_thr->th.th_team;
1302 if (!team->t.t_serialized) {
1303 if (KMP_MASTER_GTID(gtid)) {
1304 switch (__kmp_barrier_release_pattern[bt]) {
1305 case bp_hyper_bar: {
1306 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
1307 __kmp_hyper_barrier_release(bt, this_thr, gtid, tid, FALSE
1308 USE_ITT_BUILD_ARG(NULL) );
1311 case bp_hierarchical_bar: {
1312 __kmp_hierarchical_barrier_release(bt, this_thr, gtid, tid, FALSE
1313 USE_ITT_BUILD_ARG(NULL));
1317 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
1318 __kmp_tree_barrier_release(bt, this_thr, gtid, tid, FALSE
1319 USE_ITT_BUILD_ARG(NULL) );
1323 __kmp_linear_barrier_release(bt, this_thr, gtid, tid, FALSE
1324 USE_ITT_BUILD_ARG(NULL) );
1327 if (__kmp_tasking_mode != tskm_immediate_exec) {
1328 __kmp_task_team_sync(this_thr, team);
1336 __kmp_join_barrier(
int gtid)
1339 register kmp_info_t *this_thr = __kmp_threads[gtid];
1340 register kmp_team_t *team;
1341 register kmp_uint nproc;
1342 kmp_info_t *master_thread;
1348 void *itt_sync_obj = NULL;
1350 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1352 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
1358 team = this_thr->th.th_team;
1359 nproc = this_thr->th.th_team_nproc;
1360 KMP_DEBUG_ASSERT((
int)nproc == team->t.t_nproc);
1361 tid = __kmp_tid_from_gtid(gtid);
1363 team_id = team->t.t_id;
1365 master_thread = this_thr->th.th_team_master;
1367 if (master_thread != team->t.t_threads[0]) {
1368 __kmp_print_structure();
1371 KMP_DEBUG_ASSERT(master_thread == team->t.t_threads[0]);
1375 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
1376 KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team));
1377 KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root));
1378 KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]);
1379 KA_TRACE(10, (
"__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n", gtid, team_id, tid));
1383 if ((ompt_status == ompt_status_track_callback) &&
1384 ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) {
1385 ompt_callbacks.ompt_callback(ompt_event_barrier_begin)(
1386 team->t.ompt_team_info.parallel_id,
1387 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1390 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
1393 if (__kmp_tasking_mode == tskm_extra_barrier) {
1394 __kmp_tasking_barrier(team, this_thr, gtid);
1395 KA_TRACE(10, (
"__kmp_join_barrier: T#%d(%d:%d) past taking barrier\n", gtid, team_id, tid));
1398 if (__kmp_tasking_mode != tskm_immediate_exec) {
1399 KA_TRACE(20, (
"__kmp_join_barrier: T#%d, old team = %d, old task_team = %p, th_task_team = %p\n",
1400 __kmp_gtid_from_thread(this_thr), team_id, team->t.t_task_team[this_thr->th.th_task_state],
1401 this_thr->th.th_task_team));
1402 KMP_DEBUG_ASSERT(this_thr->th.th_task_team == team->t.t_task_team[this_thr->th.th_task_state]);
1410 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
1411 this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
1412 this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
1416 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1417 __kmp_itt_barrier_starting(gtid, itt_sync_obj);
1420 switch (__kmp_barrier_gather_pattern[bs_forkjoin_barrier]) {
1421 case bp_hyper_bar: {
1422 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bs_forkjoin_barrier]);
1423 __kmp_hyper_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, NULL
1424 USE_ITT_BUILD_ARG(itt_sync_obj) );
1427 case bp_hierarchical_bar: {
1428 __kmp_hierarchical_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, NULL
1429 USE_ITT_BUILD_ARG(itt_sync_obj) );
1433 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bs_forkjoin_barrier]);
1434 __kmp_tree_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, NULL
1435 USE_ITT_BUILD_ARG(itt_sync_obj) );
1439 __kmp_linear_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, NULL
1440 USE_ITT_BUILD_ARG(itt_sync_obj) );
1448 if (KMP_MASTER_TID(tid)) {
1449 if (__kmp_tasking_mode != tskm_immediate_exec) {
1452 __kmp_task_team_wait(this_thr, team
1453 USE_ITT_BUILD_ARG(itt_sync_obj) );
1456 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1457 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1460 # if USE_ITT_BUILD && USE_ITT_NOTIFY 1462 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && __kmp_forkjoin_frames_mode &&
1464 this_thr->th.th_teams_microtask == NULL &&
1466 team->t.t_active_level == 1)
1468 kmp_uint64 cur_time = __itt_get_timestamp();
1469 ident_t * loc = team->t.t_ident;
1470 kmp_info_t **other_threads = team->t.t_threads;
1471 int nproc = this_thr->th.th_team_nproc;
1473 switch(__kmp_forkjoin_frames_mode) {
1475 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
1478 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1, loc, nproc);
1481 if( __itt_metadata_add_ptr ) {
1483 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
1484 for (i=1; i<nproc; ++i) {
1485 delta += ( cur_time - other_threads[i]->th.th_bar_arrive_time );
1487 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, cur_time, delta, 0);
1489 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
1490 this_thr->th.th_frame_time = cur_time;
1498 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1499 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1504 if (KMP_MASTER_TID(tid)) {
1505 KA_TRACE(15, (
"__kmp_join_barrier: T#%d(%d:%d) says all %d team threads arrived\n",
1506 gtid, team_id, tid, nproc));
1512 KA_TRACE(10, (
"__kmp_join_barrier: T#%d(%d:%d) leaving\n", gtid, team_id, tid));
1515 if (ompt_status & ompt_status_track) {
1517 if ((ompt_status == ompt_status_track_callback) &&
1518 ompt_callbacks.ompt_callback(ompt_event_barrier_end)) {
1519 ompt_callbacks.ompt_callback(ompt_event_barrier_end)(
1520 team->t.ompt_team_info.parallel_id,
1521 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1526 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
1534 __kmp_fork_barrier(
int gtid,
int tid)
1537 kmp_info_t *this_thr = __kmp_threads[gtid];
1538 kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL;
1540 void * itt_sync_obj = NULL;
1543 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d(%d:%d) has arrived\n",
1544 gtid, (team != NULL) ? team->t.t_id : -1, tid));
1547 if (KMP_MASTER_TID(tid)) {
1548 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1549 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
1551 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 1);
1552 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1557 register kmp_info_t **other_threads = team->t.t_threads;
1563 for(i=1; i<team->t.t_nproc; ++i) {
1564 KA_TRACE(500, (
"__kmp_fork_barrier: T#%d(%d:0) checking T#%d(%d:%d) fork go == %u.\n",
1565 gtid, team->t.t_id, other_threads[i]->th.th_info.ds.ds_gtid,
1566 team->t.t_id, other_threads[i]->th.th_info.ds.ds_tid,
1567 other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go));
1568 KMP_DEBUG_ASSERT((TCR_4(other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go)
1569 & ~(KMP_BARRIER_SLEEP_STATE))
1570 == KMP_INIT_BARRIER_STATE);
1571 KMP_DEBUG_ASSERT(other_threads[i]->th.th_team == team);
1575 if (__kmp_tasking_mode != tskm_immediate_exec) {
1576 __kmp_task_team_setup(this_thr, team, 1, 0);
1583 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
1584 this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
1585 this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
1589 switch (__kmp_barrier_release_pattern[bs_forkjoin_barrier]) {
1590 case bp_hyper_bar: {
1591 KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
1592 __kmp_hyper_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE
1593 USE_ITT_BUILD_ARG(itt_sync_obj) );
1596 case bp_hierarchical_bar: {
1597 __kmp_hierarchical_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE
1598 USE_ITT_BUILD_ARG(itt_sync_obj) );
1602 KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
1603 __kmp_tree_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE
1604 USE_ITT_BUILD_ARG(itt_sync_obj) );
1608 __kmp_linear_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE
1609 USE_ITT_BUILD_ARG(itt_sync_obj) );
1614 if (TCR_4(__kmp_global.g.g_done)) {
1615 if (this_thr->th.th_task_team != NULL) {
1616 if (KMP_MASTER_TID(tid)) {
1617 TCW_PTR(this_thr->th.th_task_team, NULL);
1620 __kmp_unref_task_team(this_thr->th.th_task_team, this_thr);
1624 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1625 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
1626 if (!KMP_MASTER_TID(tid)) {
1627 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
1629 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
1633 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d is leaving early\n", gtid));
1640 team = (kmp_team_t *)TCR_PTR(this_thr->th.th_team);
1641 KMP_DEBUG_ASSERT(team != NULL);
1642 tid = __kmp_tid_from_gtid(gtid);
1645 #if KMP_BARRIER_ICV_PULL 1652 if (!KMP_MASTER_TID(tid)) {
1654 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid));
1655 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid, FALSE);
1656 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
1657 &team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs);
1660 #endif // KMP_BARRIER_ICV_PULL 1662 if (__kmp_tasking_mode != tskm_immediate_exec) {
1663 __kmp_task_team_sync(this_thr, team);
1666 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1667 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
1668 if (proc_bind == proc_bind_intel) {
1670 #if KMP_AFFINITY_SUPPORTED 1672 if(__kmp_affinity_type == affinity_balanced && team->t.t_size_changed) {
1673 __kmp_balanced_affinity(tid, team->t.t_nproc);
1675 #endif // KMP_AFFINITY_SUPPORTED 1676 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1678 else if (proc_bind != proc_bind_false) {
1679 if (this_thr->th.th_new_place == this_thr->th.th_current_place) {
1680 KA_TRACE(100, (
"__kmp_fork_barrier: T#%d already in correct place %d\n",
1681 __kmp_gtid_from_thread(this_thr), this_thr->th.th_current_place));
1684 __kmp_affinity_set_place(gtid);
1689 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1690 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
1691 if (!KMP_MASTER_TID(tid)) {
1693 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
1694 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
1698 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid, team->t.t_id, tid));
1703 __kmp_setup_icv_copy(kmp_team_t *team,
int new_nproc, kmp_internal_control_t *new_icvs,
ident_t *loc )
1707 KMP_DEBUG_ASSERT(team && new_nproc && new_icvs);
1708 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
1713 #if KMP_BARRIER_ICV_PULL 1716 KMP_DEBUG_ASSERT(team->t.t_threads[0]);
1717 copy_icvs(&team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs, new_icvs);
1718 KF_TRACE(10, (
"__kmp_setup_icv_copy: PULL: T#%d this_thread=%p team=%p\n",
1719 0, team->t.t_threads[0], team));
1720 #elif KMP_BARRIER_ICV_PUSH 1722 KF_TRACE(10, (
"__kmp_setup_icv_copy: PUSH: T#%d this_thread=%p team=%p\n",
1723 0, team->t.t_threads[0], team));
1727 KMP_DEBUG_ASSERT(team->t.t_threads[0]);
1728 for (
int f=1; f<new_nproc; ++f) {
1730 KF_TRACE(10, (
"__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
1731 f, team->t.t_threads[f], team));
1732 __kmp_init_implicit_task(loc, team->t.t_threads[f], team, f, FALSE);
1733 ngo_store_icvs(&team->t.t_implicit_task_taskdata[f].td_icvs, new_icvs);
1734 KF_TRACE(10, (
"__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
1735 f, team->t.t_threads[f], team));
1738 #endif // KMP_BARRIER_ICV_PULL #define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_STOP_EXPLICIT_TIMER(name)
"Stops" an explicit timer.
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.