20 #include "kmp_wait_release.h" 31 #ifdef KMP_SUPPORT_GRAPH_OUTPUT 32 static kmp_int32 kmp_node_id_seed = 0;
36 __kmp_init_node ( kmp_depnode_t *node )
39 node->dn.successors = NULL;
40 __kmp_init_lock(&node->dn.lock);
42 #ifdef KMP_SUPPORT_GRAPH_OUTPUT 43 node->dn.id = KMP_TEST_THEN_INC32(&kmp_node_id_seed);
47 static inline kmp_depnode_t *
48 __kmp_node_ref ( kmp_depnode_t *node )
50 KMP_TEST_THEN_INC32(&node->dn.nrefs);
55 __kmp_node_deref ( kmp_info_t *thread, kmp_depnode_t *node )
59 kmp_int32 n = KMP_TEST_THEN_DEC32(&node->dn.nrefs) - 1;
61 KMP_ASSERT(node->dn.nrefs == 0);
63 __kmp_fast_free(thread,node);
65 __kmp_thread_free(thread,node);
70 #define KMP_ACQUIRE_DEPNODE(gtid,n) __kmp_acquire_lock(&(n)->dn.lock,(gtid)) 71 #define KMP_RELEASE_DEPNODE(gtid,n) __kmp_release_lock(&(n)->dn.lock,(gtid)) 74 __kmp_depnode_list_free ( kmp_info_t *thread, kmp_depnode_list *list );
76 static const kmp_int32 kmp_dephash_log2 = 6;
77 static const kmp_int32 kmp_dephash_size = (1 << kmp_dephash_log2);
79 static inline kmp_int32
80 __kmp_dephash_hash ( kmp_intptr_t addr )
83 return ((addr >> kmp_dephash_log2) ^ addr) % kmp_dephash_size;
86 static kmp_dephash_t *
87 __kmp_dephash_create ( kmp_info_t *thread )
91 kmp_int32 size = kmp_dephash_size *
sizeof(kmp_dephash_entry_t) +
sizeof(kmp_dephash_t);
94 h = (kmp_dephash_t *) __kmp_fast_allocate( thread, size );
96 h = (kmp_dephash_t *) __kmp_thread_malloc( thread, size );
102 h->buckets = (kmp_dephash_entry **)(h+1);
104 for ( kmp_int32 i = 0; i < kmp_dephash_size; i++ )
111 __kmp_dephash_free ( kmp_info_t *thread, kmp_dephash_t *h )
113 for ( kmp_int32 i=0; i < kmp_dephash_size; i++ ) {
114 if ( h->buckets[i] ) {
115 kmp_dephash_entry_t *next;
116 for ( kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next ) {
117 next = entry->next_in_bucket;
118 __kmp_depnode_list_free(thread,entry->last_ins);
119 __kmp_node_deref(thread,entry->last_out);
121 __kmp_fast_free(thread,entry);
123 __kmp_thread_free(thread,entry);
129 __kmp_fast_free(thread,h);
131 __kmp_thread_free(thread,h);
135 static kmp_dephash_entry *
136 __kmp_dephash_find ( kmp_info_t *thread, kmp_dephash_t *h, kmp_intptr_t addr )
138 kmp_int32 bucket = __kmp_dephash_hash(addr);
140 kmp_dephash_entry_t *entry;
141 for ( entry = h->buckets[bucket]; entry; entry = entry->next_in_bucket )
142 if ( entry->addr == addr )
break;
144 if ( entry == NULL ) {
147 entry = (kmp_dephash_entry_t *) __kmp_fast_allocate( thread,
sizeof(kmp_dephash_entry_t) );
149 entry = (kmp_dephash_entry_t *) __kmp_thread_malloc( thread,
sizeof(kmp_dephash_entry_t) );
152 entry->last_out = NULL;
153 entry->last_ins = NULL;
154 entry->next_in_bucket = h->buckets[bucket];
155 h->buckets[bucket] = entry;
158 if ( entry->next_in_bucket ) h->nconflicts++;
164 static kmp_depnode_list_t *
165 __kmp_add_node ( kmp_info_t *thread, kmp_depnode_list_t *list, kmp_depnode_t *node )
167 kmp_depnode_list_t *new_head;
170 new_head = (kmp_depnode_list_t *) __kmp_fast_allocate(thread,
sizeof(kmp_depnode_list_t));
172 new_head = (kmp_depnode_list_t *) __kmp_thread_malloc(thread,
sizeof(kmp_depnode_list_t));
175 new_head->node = __kmp_node_ref(node);
176 new_head->next = list;
182 __kmp_depnode_list_free ( kmp_info_t *thread, kmp_depnode_list *list )
184 kmp_depnode_list *next;
186 for ( ; list ; list = next ) {
189 __kmp_node_deref(thread,list->node);
191 __kmp_fast_free(thread,list);
193 __kmp_thread_free(thread,list);
199 __kmp_track_dependence ( kmp_depnode_t *source, kmp_depnode_t *sink )
201 #ifdef KMP_SUPPORT_GRAPH_OUTPUT 202 kmp_taskdata_t * task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
203 kmp_taskdata_t * task_sink = KMP_TASK_TO_TASKDATA(sink->dn.task);
205 __kmp_printf(
"%d(%s) -> %d(%s)\n", source->dn.id, task_source->td_ident->psource, sink->dn.id, task_sink->td_ident->psource);
209 template<
bool filter >
210 static inline kmp_int32
211 __kmp_process_deps ( kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *hash,
212 bool dep_barrier,kmp_int32 ndeps, kmp_depend_info_t *dep_list)
214 KA_TRACE(30, (
"__kmp_process_deps<%d>: T#%d processing %d depencies : dep_barrier = %d\n", filter, gtid, ndeps, dep_barrier ) );
216 kmp_info_t *thread = __kmp_threads[ gtid ];
217 kmp_int32 npredecessors=0;
218 for ( kmp_int32 i = 0; i < ndeps ; i++ ) {
219 const kmp_depend_info_t * dep = &dep_list[i];
221 KMP_DEBUG_ASSERT(dep->flags.in);
223 if ( filter && dep->base_addr == 0 )
continue;
225 kmp_dephash_entry_t *info = __kmp_dephash_find(thread,hash,dep->base_addr);
226 kmp_depnode_t *last_out = info->last_out;
228 if ( dep->flags.out && info->last_ins ) {
229 for ( kmp_depnode_list_t * p = info->last_ins; p; p = p->next ) {
230 kmp_depnode_t * indep = p->node;
231 if ( indep->dn.task ) {
232 KMP_ACQUIRE_DEPNODE(gtid,indep);
233 if ( indep->dn.task ) {
234 __kmp_track_dependence(indep,node);
235 indep->dn.successors = __kmp_add_node(thread, indep->dn.successors, node);
236 KA_TRACE(40,(
"__kmp_process_deps<%d>: T#%d adding dependence from %p to %p",
237 filter,gtid, KMP_TASK_TO_TASKDATA(indep->dn.task), KMP_TASK_TO_TASKDATA(node->dn.task)));
240 KMP_RELEASE_DEPNODE(gtid,indep);
244 __kmp_depnode_list_free(thread,info->last_ins);
245 info->last_ins = NULL;
247 }
else if ( last_out && last_out->dn.task ) {
248 KMP_ACQUIRE_DEPNODE(gtid,last_out);
249 if ( last_out->dn.task ) {
250 __kmp_track_dependence(last_out,node);
251 last_out->dn.successors = __kmp_add_node(thread, last_out->dn.successors, node);
252 KA_TRACE(40,(
"__kmp_process_deps<%d>: T#%d adding dependence from %p to %p",
253 filter,gtid, KMP_TASK_TO_TASKDATA(last_out->dn.task), KMP_TASK_TO_TASKDATA(node->dn.task)));
257 KMP_RELEASE_DEPNODE(gtid,last_out);
263 __kmp_node_deref(thread,last_out);
264 info->last_out = NULL;
266 if ( dep->flags.out ) {
267 __kmp_node_deref(thread,last_out);
268 info->last_out = __kmp_node_ref(node);
270 info->last_ins = __kmp_add_node(thread, info->last_ins, node);
275 KA_TRACE(30, (
"__kmp_process_deps<%d>: T#%d found %d predecessors\n", filter, gtid, npredecessors ) );
277 return npredecessors;
280 #define NO_DEP_BARRIER (false) 281 #define DEP_BARRIER (true) 285 __kmp_check_deps ( kmp_int32 gtid, kmp_depnode_t *node, kmp_task_t *task, kmp_dephash_t *hash,
bool dep_barrier,
286 kmp_int32 ndeps, kmp_depend_info_t *dep_list,
287 kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list )
290 kmp_taskdata_t * taskdata;
292 taskdata = KMP_TASK_TO_TASKDATA(task);
293 KA_TRACE(20, (
"__kmp_check_deps: T#%d checking dependencies for task %p : %d possibly aliased dependencies, %d non-aliased depedencies : dep_barrier=%d .\n", gtid, taskdata, ndeps, ndeps_noalias, dep_barrier ) );
297 for ( i = 0; i < ndeps; i ++ ) {
298 if ( dep_list[i].base_addr != 0 )
299 for (
int j = i+1; j < ndeps; j++ )
300 if ( dep_list[i].base_addr == dep_list[j].base_addr ) {
301 dep_list[i].flags.in |= dep_list[j].flags.in;
302 dep_list[i].flags.out |= dep_list[j].flags.out;
303 dep_list[j].base_addr = 0;
309 node->dn.npredecessors = -1;
314 npredecessors = __kmp_process_deps<true>(gtid, node, hash, dep_barrier, ndeps, dep_list);
315 npredecessors += __kmp_process_deps<false>(gtid, node, hash, dep_barrier, ndeps_noalias, noalias_dep_list);
317 node->dn.task = task;
324 npredecessors = KMP_TEST_THEN_ADD32(&node->dn.npredecessors, npredecessors) + npredecessors;
326 KA_TRACE(20, (
"__kmp_check_deps: T#%d found %d predecessors for task %p \n", gtid, npredecessors, taskdata ) );
329 return npredecessors > 0 ?
true :
false;
333 __kmp_release_deps ( kmp_int32 gtid, kmp_taskdata_t *task )
335 kmp_info_t *thread = __kmp_threads[ gtid ];
336 kmp_depnode_t *node = task->td_depnode;
338 if ( task->td_dephash ) {
339 KA_TRACE(40, (
"__kmp_realease_deps: T#%d freeing dependencies hash of task %p.\n", gtid, task ) );
340 __kmp_dephash_free(thread,task->td_dephash);
345 KA_TRACE(20, (
"__kmp_realease_deps: T#%d notifying succesors of task %p.\n", gtid, task ) );
347 KMP_ACQUIRE_DEPNODE(gtid,node);
348 node->dn.task = NULL;
349 KMP_RELEASE_DEPNODE(gtid,node);
351 kmp_depnode_list_t *next;
352 for ( kmp_depnode_list_t *p = node->dn.successors; p; p = next ) {
353 kmp_depnode_t *successor = p->node;
354 kmp_int32 npredecessors = KMP_TEST_THEN_DEC32(&successor->dn.npredecessors) - 1;
357 if ( npredecessors == 0 ) {
359 if ( successor->dn.task ) {
360 KA_TRACE(20, (
"__kmp_realease_deps: T#%d successor %p of %p scheduled for execution.\n", gtid, successor->dn.task, task ) );
361 __kmp_omp_task(gtid,successor->dn.task,
false);
366 __kmp_node_deref(thread,p->node);
368 __kmp_fast_free(thread,p);
370 __kmp_thread_free(thread,p);
374 __kmp_node_deref(thread,node);
376 KA_TRACE(20, (
"__kmp_realease_deps: T#%d all successors of %p notified of completation\n", gtid, task ) );
395 kmp_int32 ndeps, kmp_depend_info_t *dep_list,
396 kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list )
399 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
400 KA_TRACE(10, (
"__kmpc_omp_task_with_deps(enter): T#%d loc=%p task=%p\n",
401 gtid, loc_ref, new_taskdata ) );
403 kmp_info_t *thread = __kmp_threads[ gtid ];
404 kmp_taskdata_t * current_task = thread->th.th_current_task;
406 bool serial = current_task->td_flags.team_serial || current_task->td_flags.tasking_ser || current_task->td_flags.final;
408 serial = serial && !(new_taskdata->td_flags.proxy == TASK_PROXY);
411 if ( !serial && ( ndeps > 0 || ndeps_noalias > 0 )) {
413 if ( current_task->td_dephash == NULL )
414 current_task->td_dephash = __kmp_dephash_create(thread);
417 kmp_depnode_t *node = (kmp_depnode_t *) __kmp_fast_allocate(thread,
sizeof(kmp_depnode_t));
419 kmp_depnode_t *node = (kmp_depnode_t *) __kmp_thread_malloc(thread,
sizeof(kmp_depnode_t));
422 __kmp_init_node(node);
423 new_taskdata->td_depnode = node;
425 if ( __kmp_check_deps( gtid, node, new_task, current_task->td_dephash, NO_DEP_BARRIER,
426 ndeps, dep_list, ndeps_noalias,noalias_dep_list ) ) {
427 KA_TRACE(10, (
"__kmpc_omp_task_with_deps(exit): T#%d task had blocking dependencies: " 428 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref,
430 return TASK_CURRENT_NOT_QUEUED;
434 kmp_task_team_t * task_team = thread->th.th_task_team;
435 if ( task_team && task_team->tt.tt_found_proxy_tasks )
439 KA_TRACE(10, (
"__kmpc_omp_task_with_deps(exit): T#%d ignored dependencies for task (serialized)" 440 "loc=%p task=%p\n", gtid, loc_ref, new_taskdata ) );
443 KA_TRACE(10, (
"__kmpc_omp_task_with_deps(exit): T#%d task had no blocking dependencies : " 444 "loc=%p task=%p, transferring to __kmpc_omp_task\n", gtid, loc_ref,
447 return __kmpc_omp_task(loc_ref,gtid,new_task);
463 kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list )
465 KA_TRACE(10, (
"__kmpc_omp_wait_deps(enter): T#%d loc=%p\n", gtid, loc_ref) );
467 if ( ndeps == 0 && ndeps_noalias == 0 ) {
468 KA_TRACE(10, (
"__kmpc_omp_wait_deps(exit): T#%d has no dependencies to wait upon : loc=%p\n", gtid, loc_ref) );
472 kmp_info_t *thread = __kmp_threads[ gtid ];
473 kmp_taskdata_t * current_task = thread->th.th_current_task;
478 bool ignore = current_task->td_flags.team_serial || current_task->td_flags.tasking_ser || current_task->td_flags.final;
480 ignore = ignore && thread->th.th_task_team->tt.tt_found_proxy_tasks == FALSE;
482 ignore = ignore || current_task->td_dephash == NULL;
485 KA_TRACE(10, (
"__kmpc_omp_wait_deps(exit): T#%d has no blocking dependencies : loc=%p\n", gtid, loc_ref) );
490 __kmp_init_node(&node);
492 if (!__kmp_check_deps( gtid, &node, NULL, current_task->td_dephash, DEP_BARRIER,
493 ndeps, dep_list, ndeps_noalias, noalias_dep_list )) {
494 KA_TRACE(10, (
"__kmpc_omp_wait_deps(exit): T#%d has no blocking dependencies : loc=%p\n", gtid, loc_ref) );
498 int thread_finished = FALSE;
499 kmp_flag_32 flag((
volatile kmp_uint32 *)&(node.dn.npredecessors), 0U);
500 while ( node.dn.npredecessors > 0 ) {
501 flag.execute_tasks(thread, gtid, FALSE, &thread_finished,
505 __kmp_task_stealing_constraint );
508 KA_TRACE(10, (
"__kmpc_omp_wait_deps(exit): T#%d finished waiting : loc=%p\n", gtid, loc_ref) );
void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list)
kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list)