3330 lines
113 KiB
C
3330 lines
113 KiB
C
/*
|
|
* kmp_csupport.c -- kfront linkage support for OpenMP.
|
|
*/
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// The LLVM Compiler Infrastructure
|
|
//
|
|
// This file is dual licensed under the MIT and the University of Illinois Open
|
|
// Source Licenses. See LICENSE.txt for details.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
#include "omp.h" /* extern "C" declarations of user-visible routines */
|
|
#include "kmp.h"
|
|
#include "kmp_i18n.h"
|
|
#include "kmp_itt.h"
|
|
#include "kmp_lock.h"
|
|
#include "kmp_error.h"
|
|
#include "kmp_stats.h"
|
|
|
|
#if OMPT_SUPPORT
|
|
#include "ompt-internal.h"
|
|
#include "ompt-specific.h"
|
|
#endif
|
|
|
|
#define MAX_MESSAGE 512
|
|
|
|
/* ------------------------------------------------------------------------ */
|
|
/* ------------------------------------------------------------------------ */
|
|
|
|
/* flags will be used in future, e.g., to implement */
|
|
/* openmp_strict library restrictions */
|
|
|
|
/*!
|
|
* @ingroup STARTUP_SHUTDOWN
|
|
* @param loc in source location information
|
|
* @param flags in for future use (currently ignored)
|
|
*
|
|
* Initialize the runtime library. This call is optional; if it is not made then
|
|
* it will be implicitly called by attempts to use other library functions.
|
|
*
|
|
*/
|
|
void
|
|
__kmpc_begin(ident_t *loc, kmp_int32 flags)
|
|
{
|
|
// By default __kmp_ignore_mppbeg() returns TRUE.
|
|
if (__kmp_ignore_mppbeg() == FALSE) {
|
|
__kmp_internal_begin();
|
|
|
|
KC_TRACE( 10, ("__kmpc_begin: called\n" ) );
|
|
}
|
|
}
|
|
|
|
/*!
|
|
* @ingroup STARTUP_SHUTDOWN
|
|
* @param loc source location information
|
|
*
|
|
* Shutdown the runtime library. This is also optional, and even if called will not
|
|
* do anything unless the `KMP_IGNORE_MPPEND` environment variable is set to zero.
|
|
*/
|
|
void
|
|
__kmpc_end(ident_t *loc)
|
|
{
|
|
// By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end() call no-op.
|
|
// However, this can be overridden with KMP_IGNORE_MPPEND environment variable.
|
|
// If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend() returns FALSE and __kmpc_end()
|
|
// will unregister this root (it can cause library shut down).
|
|
if (__kmp_ignore_mppend() == FALSE) {
|
|
KC_TRACE( 10, ("__kmpc_end: called\n" ) );
|
|
KA_TRACE( 30, ("__kmpc_end\n" ));
|
|
|
|
__kmp_internal_end_thread( -1 );
|
|
}
|
|
}
|
|
|
|
/*!
|
|
@ingroup THREAD_STATES
|
|
@param loc Source location information.
|
|
@return The global thread index of the active thread.
|
|
|
|
This function can be called in any context.
|
|
|
|
If the runtime has ony been entered at the outermost level from a
|
|
single (necessarily non-OpenMP<sup>*</sup>) thread, then the thread number is that
|
|
which would be returned by omp_get_thread_num() in the outermost
|
|
active parallel construct. (Or zero if there is no active parallel
|
|
construct, since the master thread is necessarily thread zero).
|
|
|
|
If multiple non-OpenMP threads all enter an OpenMP construct then this
|
|
will be a unique thread identifier among all the threads created by
|
|
the OpenMP runtime (but the value cannote be defined in terms of
|
|
OpenMP thread ids returned by omp_get_thread_num()).
|
|
|
|
*/
|
|
kmp_int32
|
|
__kmpc_global_thread_num(ident_t *loc)
|
|
{
|
|
kmp_int32 gtid = __kmp_entry_gtid();
|
|
|
|
KC_TRACE( 10, ("__kmpc_global_thread_num: T#%d\n", gtid ) );
|
|
|
|
return gtid;
|
|
}
|
|
|
|
/*!
|
|
@ingroup THREAD_STATES
|
|
@param loc Source location information.
|
|
@return The number of threads under control of the OpenMP<sup>*</sup> runtime
|
|
|
|
This function can be called in any context.
|
|
It returns the total number of threads under the control of the OpenMP runtime. That is
|
|
not a number that can be determined by any OpenMP standard calls, since the library may be
|
|
called from more than one non-OpenMP thread, and this reflects the total over all such calls.
|
|
Similarly the runtime maintains underlying threads even when they are not active (since the cost
|
|
of creating and destroying OS threads is high), this call counts all such threads even if they are not
|
|
waiting for work.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_global_num_threads(ident_t *loc)
|
|
{
|
|
KC_TRACE( 10, ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
|
|
|
|
return TCR_4(__kmp_nth);
|
|
}
|
|
|
|
/*!
|
|
@ingroup THREAD_STATES
|
|
@param loc Source location information.
|
|
@return The thread number of the calling thread in the innermost active parallel construct.
|
|
|
|
*/
|
|
kmp_int32
|
|
__kmpc_bound_thread_num(ident_t *loc)
|
|
{
|
|
KC_TRACE( 10, ("__kmpc_bound_thread_num: called\n" ) );
|
|
return __kmp_tid_from_gtid( __kmp_entry_gtid() );
|
|
}
|
|
|
|
/*!
|
|
@ingroup THREAD_STATES
|
|
@param loc Source location information.
|
|
@return The number of threads in the innermost active parallel construct.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_bound_num_threads(ident_t *loc)
|
|
{
|
|
KC_TRACE( 10, ("__kmpc_bound_num_threads: called\n" ) );
|
|
|
|
return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
|
|
}
|
|
|
|
/*!
|
|
* @ingroup DEPRECATED
|
|
* @param loc location description
|
|
*
|
|
* This function need not be called. It always returns TRUE.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_ok_to_fork(ident_t *loc)
|
|
{
|
|
#ifndef KMP_DEBUG
|
|
|
|
return TRUE;
|
|
|
|
#else
|
|
|
|
const char *semi2;
|
|
const char *semi3;
|
|
int line_no;
|
|
|
|
if (__kmp_par_range == 0) {
|
|
return TRUE;
|
|
}
|
|
semi2 = loc->psource;
|
|
if (semi2 == NULL) {
|
|
return TRUE;
|
|
}
|
|
semi2 = strchr(semi2, ';');
|
|
if (semi2 == NULL) {
|
|
return TRUE;
|
|
}
|
|
semi2 = strchr(semi2 + 1, ';');
|
|
if (semi2 == NULL) {
|
|
return TRUE;
|
|
}
|
|
if (__kmp_par_range_filename[0]) {
|
|
const char *name = semi2 - 1;
|
|
while ((name > loc->psource) && (*name != '/') && (*name != ';')) {
|
|
name--;
|
|
}
|
|
if ((*name == '/') || (*name == ';')) {
|
|
name++;
|
|
}
|
|
if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
|
|
return __kmp_par_range < 0;
|
|
}
|
|
}
|
|
semi3 = strchr(semi2 + 1, ';');
|
|
if (__kmp_par_range_routine[0]) {
|
|
if ((semi3 != NULL) && (semi3 > semi2)
|
|
&& (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
|
|
return __kmp_par_range < 0;
|
|
}
|
|
}
|
|
if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) {
|
|
if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
|
|
return __kmp_par_range > 0;
|
|
}
|
|
return __kmp_par_range < 0;
|
|
}
|
|
return TRUE;
|
|
|
|
#endif /* KMP_DEBUG */
|
|
|
|
}
|
|
|
|
/*!
|
|
@ingroup THREAD_STATES
|
|
@param loc Source location information.
|
|
@return 1 if this thread is executing inside an active parallel region, zero if not.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_in_parallel( ident_t *loc )
|
|
{
|
|
return __kmp_entry_thread() -> th.th_root -> r.r_active;
|
|
}
|
|
|
|
/*!
|
|
@ingroup PARALLEL
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
@param num_threads number of threads requested for this parallel construct
|
|
|
|
Set the number of threads to be used by the next fork spawned by this thread.
|
|
This call is only required if the parallel construct has a `num_threads` clause.
|
|
*/
|
|
void
|
|
__kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads )
|
|
{
|
|
KA_TRACE( 20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
|
|
global_tid, num_threads ) );
|
|
|
|
__kmp_push_num_threads( loc, global_tid, num_threads );
|
|
}
|
|
|
|
void
|
|
__kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid )
|
|
{
|
|
KA_TRACE( 20, ("__kmpc_pop_num_threads: enter\n" ) );
|
|
|
|
/* the num_threads are automatically popped */
|
|
}
|
|
|
|
|
|
#if OMP_40_ENABLED
|
|
|
|
void
|
|
__kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
|
|
{
|
|
KA_TRACE( 20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
|
|
global_tid, proc_bind ) );
|
|
|
|
__kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
|
|
}
|
|
|
|
#endif /* OMP_40_ENABLED */
|
|
|
|
|
|
/*!
|
|
@ingroup PARALLEL
|
|
@param loc source location information
|
|
@param argc total number of arguments in the ellipsis
|
|
@param microtask pointer to callback routine consisting of outlined parallel construct
|
|
@param ... pointers to shared variables that aren't global
|
|
|
|
Do the actual fork and call the microtask in the relevant number of threads.
|
|
*/
|
|
void
|
|
__kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
|
|
{
|
|
int gtid = __kmp_entry_gtid();
|
|
|
|
#if (KMP_STATS_ENABLED)
|
|
int inParallel = __kmpc_in_parallel(loc);
|
|
if (inParallel)
|
|
{
|
|
KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL);
|
|
}
|
|
else
|
|
{
|
|
KMP_COUNT_BLOCK(OMP_PARALLEL);
|
|
}
|
|
#endif
|
|
|
|
// maybe to save thr_state is enough here
|
|
{
|
|
va_list ap;
|
|
va_start( ap, microtask );
|
|
|
|
#if OMPT_SUPPORT
|
|
int tid = __kmp_tid_from_gtid( gtid );
|
|
kmp_info_t *master_th = __kmp_threads[ gtid ];
|
|
kmp_team_t *parent_team = master_th->th.th_team;
|
|
if (ompt_enabled) {
|
|
parent_team->t.t_implicit_task_taskdata[tid].
|
|
ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
|
|
}
|
|
#endif
|
|
|
|
#if INCLUDE_SSC_MARKS
|
|
SSC_MARK_FORKING();
|
|
#endif
|
|
__kmp_fork_call( loc, gtid, fork_context_intel,
|
|
argc,
|
|
#if OMPT_SUPPORT
|
|
VOLATILE_CAST(void *) microtask, // "unwrapped" task
|
|
#endif
|
|
VOLATILE_CAST(microtask_t) microtask, // "wrapped" task
|
|
VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
|
|
/* TODO: revert workaround for Intel(R) 64 tracker #96 */
|
|
#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
|
|
&ap
|
|
#else
|
|
ap
|
|
#endif
|
|
);
|
|
#if INCLUDE_SSC_MARKS
|
|
SSC_MARK_JOINING();
|
|
#endif
|
|
__kmp_join_call( loc, gtid
|
|
#if OMPT_SUPPORT
|
|
, fork_context_intel
|
|
#endif
|
|
);
|
|
|
|
va_end( ap );
|
|
|
|
#if OMPT_SUPPORT
|
|
if (ompt_enabled) {
|
|
parent_team->t.t_implicit_task_taskdata[tid].
|
|
ompt_task_info.frame.reenter_runtime_frame = 0;
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
|
|
#if OMP_40_ENABLED
|
|
/*!
|
|
@ingroup PARALLEL
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
@param num_teams number of teams requested for the teams construct
|
|
@param num_threads number of threads per team requested for the teams construct
|
|
|
|
Set the number of teams to be used by the teams construct.
|
|
This call is only required if the teams construct has a `num_teams` clause
|
|
or a `thread_limit` clause (or both).
|
|
*/
|
|
void
|
|
__kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads )
|
|
{
|
|
KA_TRACE( 20, ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
|
|
global_tid, num_teams, num_threads ) );
|
|
|
|
__kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
|
|
}
|
|
|
|
/*!
|
|
@ingroup PARALLEL
|
|
@param loc source location information
|
|
@param argc total number of arguments in the ellipsis
|
|
@param microtask pointer to callback routine consisting of outlined teams construct
|
|
@param ... pointers to shared variables that aren't global
|
|
|
|
Do the actual fork and call the microtask in the relevant number of threads.
|
|
*/
|
|
void
|
|
__kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
|
|
{
|
|
int gtid = __kmp_entry_gtid();
|
|
kmp_info_t *this_thr = __kmp_threads[ gtid ];
|
|
va_list ap;
|
|
va_start( ap, microtask );
|
|
|
|
KMP_COUNT_BLOCK(OMP_TEAMS);
|
|
|
|
// remember teams entry point and nesting level
|
|
this_thr->th.th_teams_microtask = microtask;
|
|
this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level; // AC: can be >0 on host
|
|
|
|
#if OMPT_SUPPORT
|
|
kmp_team_t *parent_team = this_thr->th.th_team;
|
|
int tid = __kmp_tid_from_gtid( gtid );
|
|
if (ompt_enabled) {
|
|
parent_team->t.t_implicit_task_taskdata[tid].
|
|
ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
|
|
}
|
|
#endif
|
|
|
|
// check if __kmpc_push_num_teams called, set default number of teams otherwise
|
|
if ( this_thr->th.th_teams_size.nteams == 0 ) {
|
|
__kmp_push_num_teams( loc, gtid, 0, 0 );
|
|
}
|
|
KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
|
|
KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
|
|
KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
|
|
|
|
__kmp_fork_call( loc, gtid, fork_context_intel,
|
|
argc,
|
|
#if OMPT_SUPPORT
|
|
VOLATILE_CAST(void *) microtask, // "unwrapped" task
|
|
#endif
|
|
VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task
|
|
VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
|
|
#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
|
|
&ap
|
|
#else
|
|
ap
|
|
#endif
|
|
);
|
|
__kmp_join_call( loc, gtid
|
|
#if OMPT_SUPPORT
|
|
, fork_context_intel
|
|
#endif
|
|
);
|
|
|
|
#if OMPT_SUPPORT
|
|
if (ompt_enabled) {
|
|
parent_team->t.t_implicit_task_taskdata[tid].
|
|
ompt_task_info.frame.reenter_runtime_frame = NULL;
|
|
}
|
|
#endif
|
|
|
|
this_thr->th.th_teams_microtask = NULL;
|
|
this_thr->th.th_teams_level = 0;
|
|
*(kmp_int64*)(&this_thr->th.th_teams_size) = 0L;
|
|
va_end( ap );
|
|
}
|
|
#endif /* OMP_40_ENABLED */
|
|
|
|
|
|
//
|
|
// I don't think this function should ever have been exported.
|
|
// The __kmpc_ prefix was misapplied. I'm fairly certain that no generated
|
|
// openmp code ever called it, but it's been exported from the RTL for so
|
|
// long that I'm afraid to remove the definition.
|
|
//
|
|
int
|
|
__kmpc_invoke_task_func( int gtid )
|
|
{
|
|
return __kmp_invoke_task_func( gtid );
|
|
}
|
|
|
|
/*!
|
|
@ingroup PARALLEL
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
|
|
Enter a serialized parallel construct. This interface is used to handle a
|
|
conditional parallel region, like this,
|
|
@code
|
|
#pragma omp parallel if (condition)
|
|
@endcode
|
|
when the condition is false.
|
|
*/
|
|
void
|
|
__kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
__kmp_serialized_parallel(loc, global_tid); /* The implementation is now in kmp_runtime.c so that it can share static functions with
|
|
* kmp_fork_call since the tasks to be done are similar in each case.
|
|
*/
|
|
}
|
|
|
|
/*!
|
|
@ingroup PARALLEL
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
|
|
Leave a serialized parallel construct.
|
|
*/
|
|
void
|
|
__kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
kmp_internal_control_t *top;
|
|
kmp_info_t *this_thr;
|
|
kmp_team_t *serial_team;
|
|
|
|
KC_TRACE( 10, ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
|
|
|
|
/* skip all this code for autopar serialized loops since it results in
|
|
unacceptable overhead */
|
|
if( loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR ) )
|
|
return;
|
|
|
|
// Not autopar code
|
|
if( ! TCR_4( __kmp_init_parallel ) )
|
|
__kmp_parallel_initialize();
|
|
|
|
this_thr = __kmp_threads[ global_tid ];
|
|
serial_team = this_thr->th.th_serial_team;
|
|
|
|
#if OMP_45_ENABLED
|
|
kmp_task_team_t * task_team = this_thr->th.th_task_team;
|
|
|
|
// we need to wait for the proxy tasks before finishing the thread
|
|
if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
|
|
__kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL) ); // is an ITT object needed here?
|
|
#endif
|
|
|
|
KMP_MB();
|
|
KMP_DEBUG_ASSERT( serial_team );
|
|
KMP_ASSERT( serial_team -> t.t_serialized );
|
|
KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
|
|
KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
|
|
KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
|
|
KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
|
|
|
|
/* If necessary, pop the internal control stack values and replace the team values */
|
|
top = serial_team -> t.t_control_stack_top;
|
|
if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
|
|
copy_icvs( &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs, top );
|
|
serial_team -> t.t_control_stack_top = top -> next;
|
|
__kmp_free(top);
|
|
}
|
|
|
|
//if( serial_team -> t.t_serialized > 1 )
|
|
serial_team -> t.t_level--;
|
|
|
|
/* pop dispatch buffers stack */
|
|
KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
|
|
{
|
|
dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
|
|
serial_team->t.t_dispatch->th_disp_buffer =
|
|
serial_team->t.t_dispatch->th_disp_buffer->next;
|
|
__kmp_free( disp_buffer );
|
|
}
|
|
|
|
-- serial_team -> t.t_serialized;
|
|
if ( serial_team -> t.t_serialized == 0 ) {
|
|
|
|
/* return to the parallel section */
|
|
|
|
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
|
|
if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
|
|
__kmp_clear_x87_fpu_status_word();
|
|
__kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
|
|
__kmp_load_mxcsr( &serial_team->t.t_mxcsr );
|
|
}
|
|
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
|
|
|
|
this_thr -> th.th_team = serial_team -> t.t_parent;
|
|
this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
|
|
|
|
/* restore values cached in the thread */
|
|
this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc; /* JPH */
|
|
this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0]; /* JPH */
|
|
this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
|
|
|
|
/* TODO the below shouldn't need to be adjusted for serialized teams */
|
|
this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
|
|
t.t_dispatch[ serial_team -> t.t_master_tid ];
|
|
|
|
__kmp_pop_current_task_from_thread( this_thr );
|
|
|
|
KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
|
|
this_thr -> th.th_current_task -> td_flags.executing = 1;
|
|
|
|
if ( __kmp_tasking_mode != tskm_immediate_exec ) {
|
|
// Copy the task team from the new child / old parent team to the thread.
|
|
this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
|
|
KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
|
|
global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
|
|
}
|
|
} else {
|
|
if ( __kmp_tasking_mode != tskm_immediate_exec ) {
|
|
KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
|
|
global_tid, serial_team, serial_team -> t.t_serialized ) );
|
|
}
|
|
}
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_parallel( global_tid, NULL );
|
|
}
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information.
|
|
|
|
Execute <tt>flush</tt>. This is implemented as a full memory fence. (Though
|
|
depending on the memory ordering convention obeyed by the compiler
|
|
even that may not be necessary).
|
|
*/
|
|
void
|
|
__kmpc_flush(ident_t *loc)
|
|
{
|
|
KC_TRACE( 10, ("__kmpc_flush: called\n" ) );
|
|
|
|
/* need explicit __mf() here since use volatile instead in library */
|
|
KMP_MB(); /* Flush all pending memory write invalidates. */
|
|
|
|
#if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
|
|
#if KMP_MIC
|
|
// fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used.
|
|
// We shouldn't need it, though, since the ABI rules require that
|
|
// * If the compiler generates NGO stores it also generates the fence
|
|
// * If users hand-code NGO stores they should insert the fence
|
|
// therefore no incomplete unordered stores should be visible.
|
|
#else
|
|
// C74404
|
|
// This is to address non-temporal store instructions (sfence needed).
|
|
// The clflush instruction is addressed either (mfence needed).
|
|
// Probably the non-temporal load monvtdqa instruction should also be addressed.
|
|
// mfence is a SSE2 instruction. Do not execute it if CPU is not SSE2.
|
|
if ( ! __kmp_cpuinfo.initialized ) {
|
|
__kmp_query_cpuid( & __kmp_cpuinfo );
|
|
}; // if
|
|
if ( ! __kmp_cpuinfo.sse2 ) {
|
|
// CPU cannot execute SSE2 instructions.
|
|
} else {
|
|
#if KMP_COMPILER_ICC
|
|
_mm_mfence();
|
|
#elif KMP_COMPILER_MSVC
|
|
MemoryBarrier();
|
|
#else
|
|
__sync_synchronize();
|
|
#endif // KMP_COMPILER_ICC
|
|
}; // if
|
|
#endif // KMP_MIC
|
|
#elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64)
|
|
// Nothing to see here move along
|
|
#elif KMP_ARCH_PPC64
|
|
// Nothing needed here (we have a real MB above).
|
|
#if KMP_OS_CNK
|
|
// The flushing thread needs to yield here; this prevents a
|
|
// busy-waiting thread from saturating the pipeline. flush is
|
|
// often used in loops like this:
|
|
// while (!flag) {
|
|
// #pragma omp flush(flag)
|
|
// }
|
|
// and adding the yield here is good for at least a 10x speedup
|
|
// when running >2 threads per core (on the NAS LU benchmark).
|
|
__kmp_yield(TRUE);
|
|
#endif
|
|
#else
|
|
#error Unknown or unsupported architecture
|
|
#endif
|
|
|
|
}
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid thread id.
|
|
|
|
Execute a barrier.
|
|
*/
|
|
void
|
|
__kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
KMP_COUNT_BLOCK(OMP_BARRIER);
|
|
KC_TRACE( 10, ("__kmpc_barrier: called T#%d\n", global_tid ) );
|
|
|
|
if (! TCR_4(__kmp_init_parallel))
|
|
__kmp_parallel_initialize();
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
if ( loc == 0 ) {
|
|
KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user?
|
|
}; // if
|
|
|
|
__kmp_check_barrier( global_tid, ct_barrier, loc );
|
|
}
|
|
|
|
__kmp_threads[ global_tid ]->th.th_ident = loc;
|
|
// TODO: explicit barrier_wait_id:
|
|
// this function is called when 'barrier' directive is present or
|
|
// implicit barrier at the end of a worksharing construct.
|
|
// 1) better to add a per-thread barrier counter to a thread data structure
|
|
// 2) set to 0 when a new team is created
|
|
// 4) no sync is required
|
|
|
|
__kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
|
|
}
|
|
|
|
/* The BARRIER for a MASTER section is always explicit */
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param global_tid global thread number .
|
|
@return 1 if this thread should execute the <tt>master</tt> block, 0 otherwise.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_master(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
int status = 0;
|
|
|
|
KC_TRACE( 10, ("__kmpc_master: called T#%d\n", global_tid ) );
|
|
|
|
if( ! TCR_4( __kmp_init_parallel ) )
|
|
__kmp_parallel_initialize();
|
|
|
|
if( KMP_MASTER_GTID( global_tid )) {
|
|
KMP_COUNT_BLOCK(OMP_MASTER);
|
|
KMP_PUSH_PARTITIONED_TIMER(OMP_master);
|
|
status = 1;
|
|
}
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (status) {
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_master_begin)) {
|
|
kmp_info_t *this_thr = __kmp_threads[ global_tid ];
|
|
kmp_team_t *team = this_thr -> th.th_team;
|
|
|
|
int tid = __kmp_tid_from_gtid( global_tid );
|
|
ompt_callbacks.ompt_callback(ompt_event_master_begin)(
|
|
team->t.ompt_team_info.parallel_id,
|
|
team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
if (status)
|
|
__kmp_push_sync( global_tid, ct_master, loc, NULL, 0 );
|
|
else
|
|
__kmp_check_sync( global_tid, ct_master, loc, NULL, 0 );
|
|
#else
|
|
if (status)
|
|
__kmp_push_sync( global_tid, ct_master, loc, NULL );
|
|
else
|
|
__kmp_check_sync( global_tid, ct_master, loc, NULL );
|
|
#endif
|
|
}
|
|
|
|
return status;
|
|
}
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param global_tid global thread number .
|
|
|
|
Mark the end of a <tt>master</tt> region. This should only be called by the thread
|
|
that executes the <tt>master</tt> region.
|
|
*/
|
|
void
|
|
__kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
KC_TRACE( 10, ("__kmpc_end_master: called T#%d\n", global_tid ) );
|
|
|
|
KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
|
|
KMP_POP_PARTITIONED_TIMER();
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
kmp_info_t *this_thr = __kmp_threads[ global_tid ];
|
|
kmp_team_t *team = this_thr -> th.th_team;
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_master_end)) {
|
|
int tid = __kmp_tid_from_gtid( global_tid );
|
|
ompt_callbacks.ompt_callback(ompt_event_master_end)(
|
|
team->t.ompt_team_info.parallel_id,
|
|
team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
|
|
}
|
|
#endif
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
if( global_tid < 0 )
|
|
KMP_WARNING( ThreadIdentInvalid );
|
|
|
|
if( KMP_MASTER_GTID( global_tid ))
|
|
__kmp_pop_sync( global_tid, ct_master, loc );
|
|
}
|
|
}
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param gtid global thread number.
|
|
|
|
Start execution of an <tt>ordered</tt> construct.
|
|
*/
|
|
void
|
|
__kmpc_ordered( ident_t * loc, kmp_int32 gtid )
|
|
{
|
|
int cid = 0;
|
|
kmp_info_t *th;
|
|
KMP_DEBUG_ASSERT( __kmp_init_serial );
|
|
|
|
KC_TRACE( 10, ("__kmpc_ordered: called T#%d\n", gtid ));
|
|
|
|
if (! TCR_4(__kmp_init_parallel))
|
|
__kmp_parallel_initialize();
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_ordered_prep( gtid );
|
|
// TODO: ordered_wait_id
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
th = __kmp_threads[ gtid ];
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled) {
|
|
/* OMPT state update */
|
|
th->th.ompt_thread_info.wait_id = (uint64_t) loc;
|
|
th->th.ompt_thread_info.state = ompt_state_wait_ordered;
|
|
|
|
/* OMPT event callback */
|
|
if (ompt_callbacks.ompt_callback(ompt_event_wait_ordered)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_wait_ordered)(
|
|
th->th.ompt_thread_info.wait_id);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
|
|
(*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
|
|
else
|
|
__kmp_parallel_deo( & gtid, & cid, loc );
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled) {
|
|
/* OMPT state update */
|
|
th->th.ompt_thread_info.state = ompt_state_work_parallel;
|
|
th->th.ompt_thread_info.wait_id = 0;
|
|
|
|
/* OMPT event callback */
|
|
if (ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)(
|
|
th->th.ompt_thread_info.wait_id);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_ordered_start( gtid );
|
|
#endif /* USE_ITT_BUILD */
|
|
}
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param gtid global thread number.
|
|
|
|
End execution of an <tt>ordered</tt> construct.
|
|
*/
|
|
void
|
|
__kmpc_end_ordered( ident_t * loc, kmp_int32 gtid )
|
|
{
|
|
int cid = 0;
|
|
kmp_info_t *th;
|
|
|
|
KC_TRACE( 10, ("__kmpc_end_ordered: called T#%d\n", gtid ) );
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_ordered_end( gtid );
|
|
// TODO: ordered_wait_id
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
th = __kmp_threads[ gtid ];
|
|
|
|
if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
|
|
(*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
|
|
else
|
|
__kmp_parallel_dxo( & gtid, & cid, loc );
|
|
|
|
#if OMPT_SUPPORT && OMPT_BLAME
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
|
|
th->th.ompt_thread_info.wait_id);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
static __forceinline void
|
|
__kmp_init_indirect_csptr(kmp_critical_name * crit, ident_t const * loc, kmp_int32 gtid, kmp_indirect_locktag_t tag)
|
|
{
|
|
// Pointer to the allocated indirect lock is written to crit, while indexing is ignored.
|
|
void *idx;
|
|
kmp_indirect_lock_t **lck;
|
|
lck = (kmp_indirect_lock_t **)crit;
|
|
kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
|
|
KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
|
|
KMP_SET_I_LOCK_LOCATION(ilk, loc);
|
|
KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
|
|
KA_TRACE(20, ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_creating(ilk->lock, loc);
|
|
#endif
|
|
int status = KMP_COMPARE_AND_STORE_PTR(lck, 0, ilk);
|
|
if (status == 0) {
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_destroyed(ilk->lock);
|
|
#endif
|
|
// We don't really need to destroy the unclaimed lock here since it will be cleaned up at program exit.
|
|
//KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx);
|
|
}
|
|
KMP_DEBUG_ASSERT(*lck != NULL);
|
|
}
|
|
|
|
// Fast-path acquire tas lock
|
|
#define KMP_ACQUIRE_TAS_LOCK(lock, gtid) { \
|
|
kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
|
|
if (l->lk.poll != KMP_LOCK_FREE(tas) || \
|
|
! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \
|
|
kmp_uint32 spins; \
|
|
KMP_FSYNC_PREPARE(l); \
|
|
KMP_INIT_YIELD(spins); \
|
|
if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
|
|
KMP_YIELD(TRUE); \
|
|
} else { \
|
|
KMP_YIELD_SPIN(spins); \
|
|
} \
|
|
kmp_backoff_t backoff = __kmp_spin_backoff_params; \
|
|
while (l->lk.poll != KMP_LOCK_FREE(tas) || \
|
|
! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \
|
|
__kmp_spin_backoff(&backoff); \
|
|
if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
|
|
KMP_YIELD(TRUE); \
|
|
} else { \
|
|
KMP_YIELD_SPIN(spins); \
|
|
} \
|
|
} \
|
|
} \
|
|
KMP_FSYNC_ACQUIRED(l); \
|
|
}
|
|
|
|
// Fast-path test tas lock
|
|
#define KMP_TEST_TAS_LOCK(lock, gtid, rc) { \
|
|
kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
|
|
rc = l->lk.poll == KMP_LOCK_FREE(tas) && \
|
|
KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas)); \
|
|
}
|
|
|
|
// Fast-path release tas lock
|
|
#define KMP_RELEASE_TAS_LOCK(lock, gtid) { \
|
|
TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); \
|
|
KMP_MB(); \
|
|
}
|
|
|
|
#if KMP_USE_FUTEX
|
|
|
|
# include <unistd.h>
|
|
# include <sys/syscall.h>
|
|
# ifndef FUTEX_WAIT
|
|
# define FUTEX_WAIT 0
|
|
# endif
|
|
# ifndef FUTEX_WAKE
|
|
# define FUTEX_WAKE 1
|
|
# endif
|
|
|
|
// Fast-path acquire futex lock
|
|
#define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) { \
|
|
kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
|
|
kmp_int32 gtid_code = (gtid+1) << 1; \
|
|
KMP_MB(); \
|
|
KMP_FSYNC_PREPARE(ftx); \
|
|
kmp_int32 poll_val; \
|
|
while ((poll_val = KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \
|
|
KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \
|
|
kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \
|
|
if (!cond) { \
|
|
if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, poll_val | KMP_LOCK_BUSY(1, futex))) { \
|
|
continue; \
|
|
} \
|
|
poll_val |= KMP_LOCK_BUSY(1, futex); \
|
|
} \
|
|
kmp_int32 rc; \
|
|
if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) { \
|
|
continue; \
|
|
} \
|
|
gtid_code |= 1; \
|
|
} \
|
|
KMP_FSYNC_ACQUIRED(ftx); \
|
|
}
|
|
|
|
// Fast-path test futex lock
|
|
#define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) { \
|
|
kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
|
|
if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1 << 1, futex))) { \
|
|
KMP_FSYNC_ACQUIRED(ftx); \
|
|
rc = TRUE; \
|
|
} else { \
|
|
rc = FALSE; \
|
|
} \
|
|
}
|
|
|
|
// Fast-path release futex lock
|
|
#define KMP_RELEASE_FUTEX_LOCK(lock, gtid) { \
|
|
kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
|
|
KMP_MB(); \
|
|
KMP_FSYNC_RELEASING(ftx); \
|
|
kmp_int32 poll_val = KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \
|
|
if (KMP_LOCK_STRIP(poll_val) & 1) { \
|
|
syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \
|
|
} \
|
|
KMP_MB(); \
|
|
KMP_YIELD(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \
|
|
}
|
|
|
|
#endif // KMP_USE_FUTEX
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
static kmp_user_lock_p
|
|
__kmp_get_critical_section_ptr( kmp_critical_name * crit, ident_t const * loc, kmp_int32 gtid )
|
|
{
|
|
kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
|
|
|
|
//
|
|
// Because of the double-check, the following load
|
|
// doesn't need to be volatile.
|
|
//
|
|
kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
|
|
|
|
if ( lck == NULL ) {
|
|
void * idx;
|
|
|
|
// Allocate & initialize the lock.
|
|
// Remember allocated locks in table in order to free them in __kmp_cleanup()
|
|
lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
|
|
__kmp_init_user_lock_with_checks( lck );
|
|
__kmp_set_user_lock_location( lck, loc );
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_creating( lck );
|
|
// __kmp_itt_critical_creating() should be called *before* the first usage of underlying
|
|
// lock. It is the only place where we can guarantee it. There are chances the lock will
|
|
// destroyed with no usage, but it is not a problem, because this is not real event seen
|
|
// by user but rather setting name for object (lock). See more details in kmp_itt.h.
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
//
|
|
// Use a cmpxchg instruction to slam the start of the critical
|
|
// section with the lock pointer. If another thread beat us
|
|
// to it, deallocate the lock, and use the lock that the other
|
|
// thread allocated.
|
|
//
|
|
int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
|
|
|
|
if ( status == 0 ) {
|
|
// Deallocate the lock and reload the value.
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_destroyed( lck );
|
|
// Let ITT know the lock is destroyed and the same memory location may be reused for
|
|
// another purpose.
|
|
#endif /* USE_ITT_BUILD */
|
|
__kmp_destroy_user_lock_with_checks( lck );
|
|
__kmp_user_lock_free( &idx, gtid, lck );
|
|
lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
|
|
KMP_DEBUG_ASSERT( lck != NULL );
|
|
}
|
|
}
|
|
return lck;
|
|
}
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param global_tid global thread number .
|
|
@param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or
|
|
some other suitably unique value.
|
|
|
|
Enter code protected by a `critical` construct.
|
|
This function blocks until the executing thread can enter the critical section.
|
|
*/
|
|
void
|
|
__kmpc_critical( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit )
|
|
{
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
__kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
|
|
#else
|
|
KMP_COUNT_BLOCK(OMP_CRITICAL);
|
|
KMP_TIME_PARTITIONED_BLOCK(OMP_critical_wait); /* Time spent waiting to enter the critical section */
|
|
kmp_user_lock_p lck;
|
|
|
|
KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) );
|
|
|
|
//TODO: add THR_OVHD_STATE
|
|
|
|
KMP_CHECK_USER_LOCK_INIT();
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
}
|
|
#endif
|
|
else { // ticket, queuing or drdpa
|
|
lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
|
|
}
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_push_sync( global_tid, ct_critical, loc, lck );
|
|
|
|
/* since the critical directive binds to all threads, not just
|
|
* the current team we have to check this even if we are in a
|
|
* serialized team */
|
|
/* also, even if we are the uber thread, we still have to conduct the lock,
|
|
* as we have to contend with sibling threads */
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_acquiring( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
// Value of 'crit' should be good for using as a critical_id of the critical section directive.
|
|
__kmp_acquire_user_lock_with_checks( lck, global_tid );
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_acquired( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
KMP_START_EXPLICIT_TIMER(OMP_critical);
|
|
KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid ));
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
// Converts the given hint to an internal lock implementation
|
|
static __forceinline kmp_dyna_lockseq_t
|
|
__kmp_map_hint_to_lock(uintptr_t hint)
|
|
{
|
|
#if KMP_USE_TSX
|
|
# define KMP_TSX_LOCK(seq) lockseq_##seq
|
|
#else
|
|
# define KMP_TSX_LOCK(seq) __kmp_user_lock_seq
|
|
#endif
|
|
|
|
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
|
|
# define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm)
|
|
#else
|
|
# define KMP_CPUINFO_RTM 0
|
|
#endif
|
|
|
|
// Hints that do not require further logic
|
|
if (hint & kmp_lock_hint_hle)
|
|
return KMP_TSX_LOCK(hle);
|
|
if (hint & kmp_lock_hint_rtm)
|
|
return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm): __kmp_user_lock_seq;
|
|
if (hint & kmp_lock_hint_adaptive)
|
|
return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive): __kmp_user_lock_seq;
|
|
|
|
// Rule out conflicting hints first by returning the default lock
|
|
if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
|
|
return __kmp_user_lock_seq;
|
|
if ((hint & omp_lock_hint_speculative) && (hint & omp_lock_hint_nonspeculative))
|
|
return __kmp_user_lock_seq;
|
|
|
|
// Do not even consider speculation when it appears to be contended
|
|
if (hint & omp_lock_hint_contended)
|
|
return lockseq_queuing;
|
|
|
|
// Uncontended lock without speculation
|
|
if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
|
|
return lockseq_tas;
|
|
|
|
// HLE lock for speculation
|
|
if (hint & omp_lock_hint_speculative)
|
|
return KMP_TSX_LOCK(hle);
|
|
|
|
return __kmp_user_lock_seq;
|
|
}
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param global_tid global thread number.
|
|
@param crit identity of the critical section. This could be a pointer to a lock associated with the critical section,
|
|
or some other suitably unique value.
|
|
@param hint the lock hint.
|
|
|
|
Enter code protected by a `critical` construct with a hint. The hint value is used to suggest a lock implementation.
|
|
This function blocks until the executing thread can enter the critical section unless the hint suggests use of
|
|
speculative execution and the hardware supports it.
|
|
*/
|
|
void
|
|
__kmpc_critical_with_hint( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit, uintptr_t hint )
|
|
{
|
|
KMP_COUNT_BLOCK(OMP_CRITICAL);
|
|
kmp_user_lock_p lck;
|
|
|
|
KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) );
|
|
|
|
kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
|
|
// Check if it is initialized.
|
|
if (*lk == 0) {
|
|
kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
|
|
if (KMP_IS_D_LOCK(lckseq)) {
|
|
KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(lckseq));
|
|
} else {
|
|
__kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
|
|
}
|
|
}
|
|
// Branch for accessing the actual lock object and set operation. This branching is inevitable since
|
|
// this lock initialization does not follow the normal dispatch path (lock table is not used).
|
|
if (KMP_EXTRACT_D_TAG(lk) != 0) {
|
|
lck = (kmp_user_lock_p)lk;
|
|
if (__kmp_env_consistency_check) {
|
|
__kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint));
|
|
}
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_critical_acquiring(lck);
|
|
# endif
|
|
# if KMP_USE_INLINED_TAS
|
|
if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
|
|
KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
|
|
} else
|
|
# elif KMP_USE_INLINED_FUTEX
|
|
if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
|
|
KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
|
|
} else
|
|
# endif
|
|
{
|
|
KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
|
|
}
|
|
} else {
|
|
kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
|
|
lck = ilk->lock;
|
|
if (__kmp_env_consistency_check) {
|
|
__kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint));
|
|
}
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_critical_acquiring(lck);
|
|
# endif
|
|
KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_acquired( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
|
|
KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid ));
|
|
} // __kmpc_critical_with_hint
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param global_tid global thread number .
|
|
@param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or
|
|
some other suitably unique value.
|
|
|
|
Leave a critical section, releasing any lock that was held during its execution.
|
|
*/
|
|
void
|
|
__kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
|
|
{
|
|
kmp_user_lock_p lck;
|
|
|
|
KC_TRACE( 10, ("__kmpc_end_critical: called T#%d\n", global_tid ));
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
KMP_ASSERT(lck != NULL);
|
|
if (__kmp_env_consistency_check) {
|
|
__kmp_pop_sync(global_tid, ct_critical, loc);
|
|
}
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_critical_releasing( lck );
|
|
# endif
|
|
# if KMP_USE_INLINED_TAS
|
|
if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
|
|
KMP_RELEASE_TAS_LOCK(lck, global_tid);
|
|
} else
|
|
# elif KMP_USE_INLINED_FUTEX
|
|
if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
|
|
KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
|
|
} else
|
|
# endif
|
|
{
|
|
KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
|
|
}
|
|
} else {
|
|
kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
|
|
KMP_ASSERT(ilk != NULL);
|
|
lck = ilk->lock;
|
|
if (__kmp_env_consistency_check) {
|
|
__kmp_pop_sync(global_tid, ct_critical, loc);
|
|
}
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_critical_releasing( lck );
|
|
# endif
|
|
KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
|
|
}
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
}
|
|
#endif
|
|
else { // ticket, queuing or drdpa
|
|
lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
|
|
}
|
|
|
|
KMP_ASSERT(lck != NULL);
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_sync( global_tid, ct_critical, loc );
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_critical_releasing( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
// Value of 'crit' should be good for using as a critical_id of the critical section directive.
|
|
__kmp_release_user_lock_with_checks( lck, global_tid );
|
|
|
|
#if OMPT_SUPPORT && OMPT_BLAME
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_release_critical)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_release_critical)(
|
|
(uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
KMP_POP_PARTITIONED_TIMER();
|
|
KA_TRACE( 15, ("__kmpc_end_critical: done T#%d\n", global_tid ));
|
|
}
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid thread id.
|
|
@return one if the thread should execute the master block, zero otherwise
|
|
|
|
Start execution of a combined barrier and master. The barrier is executed inside this function.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
int status;
|
|
|
|
KC_TRACE( 10, ("__kmpc_barrier_master: called T#%d\n", global_tid ) );
|
|
|
|
if (! TCR_4(__kmp_init_parallel))
|
|
__kmp_parallel_initialize();
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_check_barrier( global_tid, ct_barrier, loc );
|
|
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc;
|
|
#endif
|
|
status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
|
|
|
|
return (status != 0) ? 0 : 1;
|
|
}
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid thread id.
|
|
|
|
Complete the execution of a combined barrier and master. This function should
|
|
only be called at the completion of the <tt>master</tt> code. Other threads will
|
|
still be waiting at the barrier and this call releases them.
|
|
*/
|
|
void
|
|
__kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
KC_TRACE( 10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid ));
|
|
|
|
__kmp_end_split_barrier ( bs_plain_barrier, global_tid );
|
|
}
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid thread id.
|
|
@return one if the thread should execute the master block, zero otherwise
|
|
|
|
Start execution of a combined barrier and master(nowait) construct.
|
|
The barrier is executed inside this function.
|
|
There is no equivalent "end" function, since the
|
|
*/
|
|
kmp_int32
|
|
__kmpc_barrier_master_nowait( ident_t * loc, kmp_int32 global_tid )
|
|
{
|
|
kmp_int32 ret;
|
|
|
|
KC_TRACE( 10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
|
|
|
|
if (! TCR_4(__kmp_init_parallel))
|
|
__kmp_parallel_initialize();
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
if ( loc == 0 ) {
|
|
KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user?
|
|
}
|
|
__kmp_check_barrier( global_tid, ct_barrier, loc );
|
|
}
|
|
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc;
|
|
#endif
|
|
__kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
|
|
|
|
ret = __kmpc_master (loc, global_tid);
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
/* there's no __kmpc_end_master called; so the (stats) */
|
|
/* actions of __kmpc_end_master are done here */
|
|
|
|
if ( global_tid < 0 ) {
|
|
KMP_WARNING( ThreadIdentInvalid );
|
|
}
|
|
if (ret) {
|
|
/* only one thread should do the pop since only */
|
|
/* one did the push (see __kmpc_master()) */
|
|
|
|
__kmp_pop_sync( global_tid, ct_master, loc );
|
|
}
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/* The BARRIER for a SINGLE process section is always explicit */
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
@return One if this thread should execute the single construct, zero otherwise.
|
|
|
|
Test whether to execute a <tt>single</tt> construct.
|
|
There are no implicit barriers in the two "single" calls, rather the compiler should
|
|
introduce an explicit barrier if it is required.
|
|
*/
|
|
|
|
kmp_int32
|
|
__kmpc_single(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
|
|
|
|
if (rc) {
|
|
// We are going to execute the single statement, so we should count it.
|
|
KMP_COUNT_BLOCK(OMP_SINGLE);
|
|
KMP_PUSH_PARTITIONED_TIMER(OMP_single);
|
|
}
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
kmp_info_t *this_thr = __kmp_threads[ global_tid ];
|
|
kmp_team_t *team = this_thr -> th.th_team;
|
|
int tid = __kmp_tid_from_gtid( global_tid );
|
|
|
|
if (ompt_enabled) {
|
|
if (rc) {
|
|
if (ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)(
|
|
team->t.ompt_team_info.parallel_id,
|
|
team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id,
|
|
team->t.ompt_team_info.microtask);
|
|
}
|
|
} else {
|
|
if (ompt_callbacks.ompt_callback(ompt_event_single_others_begin)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_single_others_begin)(
|
|
team->t.ompt_team_info.parallel_id,
|
|
team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
|
|
}
|
|
this_thr->th.ompt_thread_info.state = ompt_state_wait_single;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return rc;
|
|
}
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
|
|
Mark the end of a <tt>single</tt> construct. This function should
|
|
only be called by the thread that executed the block of code protected
|
|
by the `single` construct.
|
|
*/
|
|
void
|
|
__kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
|
|
{
|
|
__kmp_exit_single( global_tid );
|
|
KMP_POP_PARTITIONED_TIMER();
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
kmp_info_t *this_thr = __kmp_threads[ global_tid ];
|
|
kmp_team_t *team = this_thr -> th.th_team;
|
|
int tid = __kmp_tid_from_gtid( global_tid );
|
|
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)(
|
|
team->t.ompt_team_info.parallel_id,
|
|
team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc Source location
|
|
@param global_tid Global thread id
|
|
|
|
Mark the end of a statically scheduled loop.
|
|
*/
|
|
void
|
|
__kmpc_for_static_fini( ident_t *loc, kmp_int32 global_tid )
|
|
{
|
|
KE_TRACE( 10, ("__kmpc_for_static_fini called T#%d\n", global_tid));
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_loop_end)) {
|
|
ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
|
|
ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
|
|
ompt_callbacks.ompt_callback(ompt_event_loop_end)(
|
|
team_info->parallel_id, task_info->task_id);
|
|
}
|
|
#endif
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_workshare( global_tid, ct_pdo, loc );
|
|
}
|
|
|
|
/*
|
|
* User routines which take C-style arguments (call by value)
|
|
* different from the Fortran equivalent routines
|
|
*/
|
|
|
|
void
|
|
ompc_set_num_threads( int arg )
|
|
{
|
|
// !!!!! TODO: check the per-task binding
|
|
__kmp_set_num_threads( arg, __kmp_entry_gtid() );
|
|
}
|
|
|
|
void
|
|
ompc_set_dynamic( int flag )
|
|
{
|
|
kmp_info_t *thread;
|
|
|
|
/* For the thread-private implementation of the internal controls */
|
|
thread = __kmp_entry_thread();
|
|
|
|
__kmp_save_internal_controls( thread );
|
|
|
|
set__dynamic( thread, flag ? TRUE : FALSE );
|
|
}
|
|
|
|
void
|
|
ompc_set_nested( int flag )
|
|
{
|
|
kmp_info_t *thread;
|
|
|
|
/* For the thread-private internal controls implementation */
|
|
thread = __kmp_entry_thread();
|
|
|
|
__kmp_save_internal_controls( thread );
|
|
|
|
set__nested( thread, flag ? TRUE : FALSE );
|
|
}
|
|
|
|
void
|
|
ompc_set_max_active_levels( int max_active_levels )
|
|
{
|
|
/* TO DO */
|
|
/* we want per-task implementation of this internal control */
|
|
|
|
/* For the per-thread internal controls implementation */
|
|
__kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
|
|
}
|
|
|
|
void
|
|
ompc_set_schedule( omp_sched_t kind, int modifier )
|
|
{
|
|
// !!!!! TODO: check the per-task binding
|
|
__kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
|
|
}
|
|
|
|
int
|
|
ompc_get_ancestor_thread_num( int level )
|
|
{
|
|
return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
|
|
}
|
|
|
|
int
|
|
ompc_get_team_size( int level )
|
|
{
|
|
return __kmp_get_team_size( __kmp_entry_gtid(), level );
|
|
}
|
|
|
|
void
|
|
kmpc_set_stacksize( int arg )
|
|
{
|
|
// __kmp_aux_set_stacksize initializes the library if needed
|
|
__kmp_aux_set_stacksize( arg );
|
|
}
|
|
|
|
void
|
|
kmpc_set_stacksize_s( size_t arg )
|
|
{
|
|
// __kmp_aux_set_stacksize initializes the library if needed
|
|
__kmp_aux_set_stacksize( arg );
|
|
}
|
|
|
|
void
|
|
kmpc_set_blocktime( int arg )
|
|
{
|
|
int gtid, tid;
|
|
kmp_info_t *thread;
|
|
|
|
gtid = __kmp_entry_gtid();
|
|
tid = __kmp_tid_from_gtid(gtid);
|
|
thread = __kmp_thread_from_gtid(gtid);
|
|
|
|
__kmp_aux_set_blocktime( arg, thread, tid );
|
|
}
|
|
|
|
void
|
|
kmpc_set_library( int arg )
|
|
{
|
|
// __kmp_user_set_library initializes the library if needed
|
|
__kmp_user_set_library( (enum library_type)arg );
|
|
}
|
|
|
|
void
|
|
kmpc_set_defaults( char const * str )
|
|
{
|
|
// __kmp_aux_set_defaults initializes the library if needed
|
|
__kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
|
|
}
|
|
|
|
void
|
|
kmpc_set_disp_num_buffers( int arg )
|
|
{
|
|
// ignore after initialization because some teams have already
|
|
// allocated dispatch buffers
|
|
if( __kmp_init_serial == 0 && arg > 0 )
|
|
__kmp_dispatch_num_buffers = arg;
|
|
}
|
|
|
|
int
|
|
kmpc_set_affinity_mask_proc( int proc, void **mask )
|
|
{
|
|
#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
|
|
return -1;
|
|
#else
|
|
if ( ! TCR_4(__kmp_init_middle) ) {
|
|
__kmp_middle_initialize();
|
|
}
|
|
return __kmp_aux_set_affinity_mask_proc( proc, mask );
|
|
#endif
|
|
}
|
|
|
|
int
|
|
kmpc_unset_affinity_mask_proc( int proc, void **mask )
|
|
{
|
|
#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
|
|
return -1;
|
|
#else
|
|
if ( ! TCR_4(__kmp_init_middle) ) {
|
|
__kmp_middle_initialize();
|
|
}
|
|
return __kmp_aux_unset_affinity_mask_proc( proc, mask );
|
|
#endif
|
|
}
|
|
|
|
int
|
|
kmpc_get_affinity_mask_proc( int proc, void **mask )
|
|
{
|
|
#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
|
|
return -1;
|
|
#else
|
|
if ( ! TCR_4(__kmp_init_middle) ) {
|
|
__kmp_middle_initialize();
|
|
}
|
|
return __kmp_aux_get_affinity_mask_proc( proc, mask );
|
|
#endif
|
|
}
|
|
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
/*!
|
|
@ingroup THREADPRIVATE
|
|
@param loc source location information
|
|
@param gtid global thread number
|
|
@param cpy_size size of the cpy_data buffer
|
|
@param cpy_data pointer to data to be copied
|
|
@param cpy_func helper function to call for copying data
|
|
@param didit flag variable: 1=single thread; 0=not single thread
|
|
|
|
__kmpc_copyprivate implements the interface for the private data broadcast needed for
|
|
the copyprivate clause associated with a single region in an OpenMP<sup>*</sup> program (both C and Fortran).
|
|
All threads participating in the parallel region call this routine.
|
|
One of the threads (called the single thread) should have the <tt>didit</tt> variable set to 1
|
|
and all other threads should have that variable set to 0.
|
|
All threads pass a pointer to a data buffer (cpy_data) that they have built.
|
|
|
|
The OpenMP specification forbids the use of nowait on the single region when a copyprivate
|
|
clause is present. However, @ref __kmpc_copyprivate implements a barrier internally to avoid
|
|
race conditions, so the code generation for the single region should avoid generating a barrier
|
|
after the call to @ref __kmpc_copyprivate.
|
|
|
|
The <tt>gtid</tt> parameter is the global thread id for the current thread.
|
|
The <tt>loc</tt> parameter is a pointer to source location information.
|
|
|
|
Internal implementation: The single thread will first copy its descriptor address (cpy_data)
|
|
to a team-private location, then the other threads will each call the function pointed to by
|
|
the parameter cpy_func, which carries out the copy by copying the data using the cpy_data buffer.
|
|
|
|
The cpy_func routine used for the copy and the contents of the data area defined by cpy_data
|
|
and cpy_size may be built in any fashion that will allow the copy to be done. For instance,
|
|
the cpy_data buffer can hold the actual data to be copied or it may hold a list of pointers
|
|
to the data. The cpy_func routine must interpret the cpy_data buffer appropriately.
|
|
|
|
The interface to cpy_func is as follows:
|
|
@code
|
|
void cpy_func( void *destination, void *source )
|
|
@endcode
|
|
where void *destination is the cpy_data pointer for the thread being copied to
|
|
and void *source is the cpy_data pointer for the thread being copied from.
|
|
*/
|
|
void
|
|
__kmpc_copyprivate( ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void*,void*), kmp_int32 didit )
|
|
{
|
|
void **data_ptr;
|
|
|
|
KC_TRACE( 10, ("__kmpc_copyprivate: called T#%d\n", gtid ));
|
|
|
|
KMP_MB();
|
|
|
|
data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
if ( loc == 0 ) {
|
|
KMP_WARNING( ConstructIdentInvalid );
|
|
}
|
|
}
|
|
|
|
/* ToDo: Optimize the following two barriers into some kind of split barrier */
|
|
|
|
if (didit) *data_ptr = cpy_data;
|
|
|
|
/* This barrier is not a barrier region boundary */
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[gtid]->th.th_ident = loc;
|
|
#endif
|
|
__kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
|
|
|
|
if (! didit) (*cpy_func)( cpy_data, *data_ptr );
|
|
|
|
/* Consider next barrier the user-visible barrier for barrier region boundaries */
|
|
/* Nesting checks are already handled by the single construct checks */
|
|
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g. tasks can overwrite the location)
|
|
#endif
|
|
__kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
|
|
}
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
#define INIT_LOCK __kmp_init_user_lock_with_checks
|
|
#define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
|
|
#define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
|
|
#define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
|
|
#define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
|
|
#define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed
|
|
#define RELEASE_LOCK __kmp_release_user_lock_with_checks
|
|
#define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
|
|
#define TEST_LOCK __kmp_test_user_lock_with_checks
|
|
#define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
|
|
#define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
|
|
#define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
|
|
|
|
|
|
/*
|
|
* TODO: Make check abort messages use location info & pass it
|
|
* into with_checks routines
|
|
*/
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
// internal lock initializer
|
|
static __forceinline void
|
|
__kmp_init_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq)
|
|
{
|
|
if (KMP_IS_D_LOCK(seq)) {
|
|
KMP_INIT_D_LOCK(lock, seq);
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
|
|
#endif
|
|
} else {
|
|
KMP_INIT_I_LOCK(lock, seq);
|
|
#if USE_ITT_BUILD
|
|
kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
|
|
__kmp_itt_lock_creating(ilk->lock, loc);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
// internal nest lock initializer
|
|
static __forceinline void
|
|
__kmp_init_nest_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq)
|
|
{
|
|
#if KMP_USE_TSX
|
|
// Don't have nested lock implementation for speculative locks
|
|
if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
|
|
seq = __kmp_user_lock_seq;
|
|
#endif
|
|
switch (seq) {
|
|
case lockseq_tas:
|
|
seq = lockseq_nested_tas;
|
|
break;
|
|
#if KMP_USE_FUTEX
|
|
case lockseq_futex:
|
|
seq = lockseq_nested_futex;
|
|
break;
|
|
#endif
|
|
case lockseq_ticket:
|
|
seq = lockseq_nested_ticket;
|
|
break;
|
|
case lockseq_queuing:
|
|
seq = lockseq_nested_queuing;
|
|
break;
|
|
case lockseq_drdpa:
|
|
seq = lockseq_nested_drdpa;
|
|
break;
|
|
default:
|
|
seq = lockseq_nested_queuing;
|
|
}
|
|
KMP_INIT_I_LOCK(lock, seq);
|
|
#if USE_ITT_BUILD
|
|
kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
|
|
__kmp_itt_lock_creating(ilk->lock, loc);
|
|
#endif
|
|
}
|
|
|
|
/* initialize the lock with a hint */
|
|
void
|
|
__kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
|
|
{
|
|
KMP_DEBUG_ASSERT(__kmp_init_serial);
|
|
if (__kmp_env_consistency_check && user_lock == NULL) {
|
|
KMP_FATAL(LockIsUninitialized, "omp_init_lock_with_hint");
|
|
}
|
|
|
|
__kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
|
|
}
|
|
|
|
/* initialize the lock with a hint */
|
|
void
|
|
__kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
|
|
{
|
|
KMP_DEBUG_ASSERT(__kmp_init_serial);
|
|
if (__kmp_env_consistency_check && user_lock == NULL) {
|
|
KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock_with_hint");
|
|
}
|
|
|
|
__kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
|
|
}
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
|
|
/* initialize the lock */
|
|
void
|
|
__kmpc_init_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
KMP_DEBUG_ASSERT(__kmp_init_serial);
|
|
if (__kmp_env_consistency_check && user_lock == NULL) {
|
|
KMP_FATAL(LockIsUninitialized, "omp_init_lock");
|
|
}
|
|
__kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
static char const * const func = "omp_init_lock";
|
|
kmp_user_lock_p lck;
|
|
KMP_DEBUG_ASSERT( __kmp_init_serial );
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
if ( user_lock == NULL ) {
|
|
KMP_FATAL( LockIsUninitialized, func );
|
|
}
|
|
}
|
|
|
|
KMP_CHECK_USER_LOCK_INIT();
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
|
|
}
|
|
INIT_LOCK( lck );
|
|
__kmp_set_user_lock_location( lck, loc );
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_init_lock)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_init_lock)((uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_creating( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
} // __kmpc_init_lock
|
|
|
|
/* initialize the lock */
|
|
void
|
|
__kmpc_init_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
KMP_DEBUG_ASSERT(__kmp_init_serial);
|
|
if (__kmp_env_consistency_check && user_lock == NULL) {
|
|
KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock");
|
|
}
|
|
__kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
static char const * const func = "omp_init_nest_lock";
|
|
kmp_user_lock_p lck;
|
|
KMP_DEBUG_ASSERT( __kmp_init_serial );
|
|
|
|
if ( __kmp_env_consistency_check ) {
|
|
if ( user_lock == NULL ) {
|
|
KMP_FATAL( LockIsUninitialized, func );
|
|
}
|
|
}
|
|
|
|
KMP_CHECK_USER_LOCK_INIT();
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
|
|
+ sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
|
|
<= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
|
|
}
|
|
|
|
INIT_NESTED_LOCK( lck );
|
|
__kmp_set_user_lock_location( lck, loc );
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)((uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_creating( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
} // __kmpc_init_nest_lock
|
|
|
|
void
|
|
__kmpc_destroy_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
# if USE_ITT_BUILD
|
|
kmp_user_lock_p lck;
|
|
if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
|
|
lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
|
|
} else {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
__kmp_itt_lock_destroyed(lck);
|
|
# endif
|
|
KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
|
|
#else
|
|
kmp_user_lock_p lck;
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_destroy_lock" );
|
|
}
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_destroy_lock)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_destroy_lock)((uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_destroyed( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
DESTROY_LOCK( lck );
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
;
|
|
}
|
|
#endif
|
|
else {
|
|
__kmp_user_lock_free( user_lock, gtid, lck );
|
|
}
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
} // __kmpc_destroy_lock
|
|
|
|
/* destroy the lock */
|
|
void
|
|
__kmpc_destroy_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
# if USE_ITT_BUILD
|
|
kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
|
|
__kmp_itt_lock_destroyed(ilk->lock);
|
|
# endif
|
|
KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_user_lock_p lck;
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
|
|
+ sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
|
|
<= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_destroy_nest_lock" );
|
|
}
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)((uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_destroyed( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
DESTROY_NESTED_LOCK( lck );
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
|
|
+ sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
|
|
<= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
;
|
|
}
|
|
#endif
|
|
else {
|
|
__kmp_user_lock_free( user_lock, gtid, lck );
|
|
}
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
} // __kmpc_destroy_nest_lock
|
|
|
|
void
|
|
__kmpc_set_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
|
|
KMP_COUNT_BLOCK(OMP_set_lock);
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
int tag = KMP_EXTRACT_D_TAG(user_lock);
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); // itt function will get to the right lock object.
|
|
# endif
|
|
# if KMP_USE_INLINED_TAS
|
|
if (tag == locktag_tas && !__kmp_env_consistency_check) {
|
|
KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
|
|
} else
|
|
# elif KMP_USE_INLINED_FUTEX
|
|
if (tag == locktag_futex && !__kmp_env_consistency_check) {
|
|
KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
|
|
} else
|
|
# endif
|
|
{
|
|
__kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
|
|
}
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_user_lock_p lck;
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_set_lock" );
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
ACQUIRE_LOCK( lck, gtid );
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquired( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_acquired_lock)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_acquired_lock)((uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
void
|
|
__kmpc_set_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
|
|
#endif
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled) {
|
|
// missing support here: need to know whether acquired first or not
|
|
}
|
|
#endif
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
int acquire_status;
|
|
kmp_user_lock_p lck;
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
|
|
+ sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
|
|
<= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_set_nest_lock" );
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
ACQUIRE_NESTED_LOCK( lck, gtid, &acquire_status );
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquired( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
#if OMPT_SUPPORT && OMPT_TRACE
|
|
if (ompt_enabled) {
|
|
if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
|
|
if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first))
|
|
ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first)((uint64_t) lck);
|
|
} else {
|
|
if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next))
|
|
ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next)((uint64_t) lck);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
void
|
|
__kmpc_unset_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
|
|
{
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
int tag = KMP_EXTRACT_D_TAG(user_lock);
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
# if KMP_USE_INLINED_TAS
|
|
if (tag == locktag_tas && !__kmp_env_consistency_check) {
|
|
KMP_RELEASE_TAS_LOCK(user_lock, gtid);
|
|
} else
|
|
# elif KMP_USE_INLINED_FUTEX
|
|
if (tag == locktag_futex && !__kmp_env_consistency_check) {
|
|
KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
|
|
} else
|
|
# endif
|
|
{
|
|
__kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
|
|
}
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_user_lock_p lck;
|
|
|
|
/* Can't use serial interval since not block structured */
|
|
/* release the lock */
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
|
|
// "fast" path implemented to fix customer performance issue
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
|
|
#endif /* USE_ITT_BUILD */
|
|
TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
|
|
KMP_MB();
|
|
return;
|
|
#else
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
#endif
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_unset_lock" );
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_releasing( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
RELEASE_LOCK( lck, gtid );
|
|
|
|
#if OMPT_SUPPORT && OMPT_BLAME
|
|
if (ompt_enabled &&
|
|
ompt_callbacks.ompt_callback(ompt_event_release_lock)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_release_lock)((uint64_t) lck);
|
|
}
|
|
#endif
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
/* release the lock */
|
|
void
|
|
__kmpc_unset_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
|
|
{
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_user_lock_p lck;
|
|
|
|
/* Can't use serial interval since not block structured */
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
|
|
+ sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
|
|
// "fast" path implemented to fix customer performance issue
|
|
kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
|
|
#endif /* USE_ITT_BUILD */
|
|
if ( --(tl->lk.depth_locked) == 0 ) {
|
|
TCW_4(tl->lk.poll, 0);
|
|
}
|
|
KMP_MB();
|
|
return;
|
|
#else
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
#endif
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
|
|
<= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_unset_nest_lock" );
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_releasing( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
int release_status;
|
|
release_status = RELEASE_NESTED_LOCK( lck, gtid );
|
|
#if OMPT_SUPPORT && OMPT_BLAME
|
|
if (ompt_enabled) {
|
|
if (release_status == KMP_LOCK_RELEASED) {
|
|
if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)(
|
|
(uint64_t) lck);
|
|
}
|
|
} else if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)) {
|
|
ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)(
|
|
(uint64_t) lck);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
/* try to acquire the lock */
|
|
int
|
|
__kmpc_test_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
|
|
{
|
|
KMP_COUNT_BLOCK(OMP_test_lock);
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
int rc;
|
|
int tag = KMP_EXTRACT_D_TAG(user_lock);
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
# if KMP_USE_INLINED_TAS
|
|
if (tag == locktag_tas && !__kmp_env_consistency_check) {
|
|
KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
|
|
} else
|
|
# elif KMP_USE_INLINED_FUTEX
|
|
if (tag == locktag_futex && !__kmp_env_consistency_check) {
|
|
KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
|
|
} else
|
|
# endif
|
|
{
|
|
rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
|
|
}
|
|
if (rc) {
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
return FTN_TRUE;
|
|
} else {
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
return FTN_FALSE;
|
|
}
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_user_lock_p lck;
|
|
int rc;
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas )
|
|
&& ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_test_lock" );
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
rc = TEST_LOCK( lck, gtid );
|
|
#if USE_ITT_BUILD
|
|
if ( rc ) {
|
|
__kmp_itt_lock_acquired( lck );
|
|
} else {
|
|
__kmp_itt_lock_cancelled( lck );
|
|
}
|
|
#endif /* USE_ITT_BUILD */
|
|
return ( rc ? FTN_TRUE : FTN_FALSE );
|
|
|
|
/* Can't use serial interval since not block structured */
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
/* try to acquire the lock */
|
|
int
|
|
__kmpc_test_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
|
|
{
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
int rc;
|
|
# if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
|
|
# endif
|
|
rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
|
|
# if USE_ITT_BUILD
|
|
if (rc) {
|
|
__kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
|
|
} else {
|
|
__kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
|
|
}
|
|
# endif
|
|
return rc;
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_user_lock_p lck;
|
|
int rc;
|
|
|
|
if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
|
|
+ sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#if KMP_USE_FUTEX
|
|
else if ( ( __kmp_user_lock_kind == lk_futex )
|
|
&& ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
|
|
<= OMP_NEST_LOCK_T_SIZE ) ) {
|
|
lck = (kmp_user_lock_p)user_lock;
|
|
}
|
|
#endif
|
|
else {
|
|
lck = __kmp_lookup_user_lock( user_lock, "omp_test_nest_lock" );
|
|
}
|
|
|
|
#if USE_ITT_BUILD
|
|
__kmp_itt_lock_acquiring( lck );
|
|
#endif /* USE_ITT_BUILD */
|
|
|
|
rc = TEST_NESTED_LOCK( lck, gtid );
|
|
#if USE_ITT_BUILD
|
|
if ( rc ) {
|
|
__kmp_itt_lock_acquired( lck );
|
|
} else {
|
|
__kmp_itt_lock_cancelled( lck );
|
|
}
|
|
#endif /* USE_ITT_BUILD */
|
|
return rc;
|
|
|
|
/* Can't use serial interval since not block structured */
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
|
|
/*--------------------------------------------------------------------------------------------------------------------*/
|
|
|
|
/*
|
|
* Interface to fast scalable reduce methods routines
|
|
*/
|
|
|
|
// keep the selected method in a thread local structure for cross-function usage: will be used in __kmpc_end_reduce* functions;
|
|
// another solution: to re-determine the method one more time in __kmpc_end_reduce* functions (new prototype required then)
|
|
// AT: which solution is better?
|
|
#define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \
|
|
( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) )
|
|
|
|
#define __KMP_GET_REDUCTION_METHOD(gtid) \
|
|
( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method )
|
|
|
|
// description of the packed_reduction_method variable: look at the macros in kmp.h
|
|
|
|
|
|
// used in a critical section reduce block
|
|
static __forceinline void
|
|
__kmp_enter_critical_section_reduce_block( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
|
|
|
|
// this lock was visible to a customer and to the threading profile tool as a serial overhead span
|
|
// (although it's used for an internal purpose only)
|
|
// why was it visible in previous implementation?
|
|
// should we keep it visible in new reduce block?
|
|
kmp_user_lock_p lck;
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
|
|
// Check if it is initialized.
|
|
if (*lk == 0) {
|
|
if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
|
|
KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(__kmp_user_lock_seq));
|
|
} else {
|
|
__kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(__kmp_user_lock_seq));
|
|
}
|
|
}
|
|
// Branch for accessing the actual lock object and set operation. This branching is inevitable since
|
|
// this lock initialization does not follow the normal dispatch path (lock table is not used).
|
|
if (KMP_EXTRACT_D_TAG(lk) != 0) {
|
|
lck = (kmp_user_lock_p)lk;
|
|
KMP_DEBUG_ASSERT(lck != NULL);
|
|
if (__kmp_env_consistency_check) {
|
|
__kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
|
|
}
|
|
KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
|
|
} else {
|
|
kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
|
|
lck = ilk->lock;
|
|
KMP_DEBUG_ASSERT(lck != NULL);
|
|
if (__kmp_env_consistency_check) {
|
|
__kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
|
|
}
|
|
KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
|
|
}
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
// We know that the fast reduction code is only emitted by Intel compilers
|
|
// with 32 byte critical sections. If there isn't enough space, then we
|
|
// have to use a pointer.
|
|
if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
}
|
|
else {
|
|
lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
|
|
}
|
|
KMP_DEBUG_ASSERT( lck != NULL );
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_push_sync( global_tid, ct_critical, loc, lck );
|
|
|
|
__kmp_acquire_user_lock_with_checks( lck, global_tid );
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
}
|
|
|
|
// used in a critical section reduce block
|
|
static __forceinline void
|
|
__kmp_end_critical_section_reduce_block( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
|
|
|
|
kmp_user_lock_p lck;
|
|
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
|
|
if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
|
|
lck = (kmp_user_lock_p)crit;
|
|
if (__kmp_env_consistency_check)
|
|
__kmp_pop_sync(global_tid, ct_critical, loc);
|
|
KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
|
|
} else {
|
|
kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
|
|
if (__kmp_env_consistency_check)
|
|
__kmp_pop_sync(global_tid, ct_critical, loc);
|
|
KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
|
|
}
|
|
|
|
#else // KMP_USE_DYNAMIC_LOCK
|
|
|
|
// We know that the fast reduction code is only emitted by Intel compilers with 32 byte critical
|
|
// sections. If there isn't enough space, then we have to use a pointer.
|
|
if ( __kmp_base_user_lock_size > 32 ) {
|
|
lck = *( (kmp_user_lock_p *) crit );
|
|
KMP_ASSERT( lck != NULL );
|
|
} else {
|
|
lck = (kmp_user_lock_p) crit;
|
|
}
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_sync( global_tid, ct_critical, loc );
|
|
|
|
__kmp_release_user_lock_with_checks( lck, global_tid );
|
|
|
|
#endif // KMP_USE_DYNAMIC_LOCK
|
|
} // __kmp_end_critical_section_reduce_block
|
|
|
|
|
|
/* 2.a.i. Reduce Block without a terminating barrier */
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
@param num_vars number of items (variables) to be reduced
|
|
@param reduce_size size of data in bytes to be reduced
|
|
@param reduce_data pointer to data to be reduced
|
|
@param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data
|
|
@param lck pointer to the unique lock data structure
|
|
@result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed
|
|
|
|
The nowait version is used for a reduce clause with the nowait argument.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_reduce_nowait(
|
|
ident_t *loc, kmp_int32 global_tid,
|
|
kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
|
|
kmp_critical_name *lck ) {
|
|
|
|
KMP_COUNT_BLOCK(REDUCE_nowait);
|
|
int retval = 0;
|
|
PACKED_REDUCTION_METHOD_T packed_reduction_method;
|
|
#if OMP_40_ENABLED
|
|
kmp_team_t *team;
|
|
kmp_info_t *th;
|
|
int teams_swapped = 0, task_state;
|
|
#endif
|
|
KA_TRACE( 10, ( "__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
|
|
|
|
// why do we need this initialization here at all?
|
|
// Reduction clause can not be used as a stand-alone directive.
|
|
|
|
// do not call __kmp_serial_initialize(), it will be called by __kmp_parallel_initialize() if needed
|
|
// possible detection of false-positive race by the threadchecker ???
|
|
if( ! TCR_4( __kmp_init_parallel ) )
|
|
__kmp_parallel_initialize();
|
|
|
|
// check correctness of reduce block nesting
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
|
|
#else
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_push_sync( global_tid, ct_reduce, loc, NULL );
|
|
#endif
|
|
|
|
#if OMP_40_ENABLED
|
|
th = __kmp_thread_from_gtid(global_tid);
|
|
if( th->th.th_teams_microtask ) { // AC: check if we are inside the teams construct?
|
|
team = th->th.th_team;
|
|
if( team->t.t_level == th->th.th_teams_level ) {
|
|
// this is reduction at teams construct
|
|
KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid); // AC: check that tid == 0
|
|
// Let's swap teams temporarily for the reduction barrier
|
|
teams_swapped = 1;
|
|
th->th.th_info.ds.ds_tid = team->t.t_master_tid;
|
|
th->th.th_team = team->t.t_parent;
|
|
th->th.th_team_nproc = th->th.th_team->t.t_nproc;
|
|
th->th.th_task_team = th->th.th_team->t.t_task_team[0];
|
|
task_state = th->th.th_task_state;
|
|
th->th.th_task_state = 0;
|
|
}
|
|
}
|
|
#endif // OMP_40_ENABLED
|
|
|
|
// packed_reduction_method value will be reused by __kmp_end_reduce* function, the value should be kept in a variable
|
|
// the variable should be either a construct-specific or thread-specific property, not a team specific property
|
|
// (a thread can reach the next reduce block on the next construct, reduce method may differ on the next construct)
|
|
// an ident_t "loc" parameter could be used as a construct-specific property (what if loc == 0?)
|
|
// (if both construct-specific and team-specific variables were shared, then unness extra syncs should be needed)
|
|
// a thread-specific variable is better regarding two issues above (next construct and extra syncs)
|
|
// a thread-specific "th_local.reduction_method" variable is used currently
|
|
// each thread executes 'determine' and 'set' lines (no need to execute by one thread, to avoid unness extra syncs)
|
|
|
|
packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
|
|
__KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
|
|
|
|
if( packed_reduction_method == critical_reduce_block ) {
|
|
|
|
__kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
|
|
retval = 1;
|
|
|
|
} else if( packed_reduction_method == empty_reduce_block ) {
|
|
|
|
// usage: if team size == 1, no synchronization is required ( Intel platforms only )
|
|
retval = 1;
|
|
|
|
} else if( packed_reduction_method == atomic_reduce_block ) {
|
|
|
|
retval = 2;
|
|
|
|
// all threads should do this pop here (because __kmpc_end_reduce_nowait() won't be called by the code gen)
|
|
// (it's not quite good, because the checking block has been closed by this 'pop',
|
|
// but atomic operation has not been executed yet, will be executed slightly later, literally on next instruction)
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_sync( global_tid, ct_reduce, loc );
|
|
|
|
} else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
|
|
|
|
//AT: performance issue: a real barrier here
|
|
//AT: (if master goes slow, other threads are blocked here waiting for the master to come and release them)
|
|
//AT: (it's not what a customer might expect specifying NOWAIT clause)
|
|
//AT: (specifying NOWAIT won't result in improvement of performance, it'll be confusing to a customer)
|
|
//AT: another implementation of *barrier_gather*nowait() (or some other design) might go faster
|
|
// and be more in line with sense of NOWAIT
|
|
//AT: TO DO: do epcc test and compare times
|
|
|
|
// this barrier should be invisible to a customer and to the threading profile tool
|
|
// (it's neither a terminating barrier nor customer's code, it's used for an internal purpose)
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc;
|
|
#endif
|
|
retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
|
|
retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
|
|
|
|
// all other workers except master should do this pop here
|
|
// ( none of other workers will get to __kmpc_end_reduce_nowait() )
|
|
if ( __kmp_env_consistency_check ) {
|
|
if( retval == 0 ) {
|
|
__kmp_pop_sync( global_tid, ct_reduce, loc );
|
|
}
|
|
}
|
|
|
|
} else {
|
|
|
|
// should never reach this block
|
|
KMP_ASSERT( 0 ); // "unexpected method"
|
|
|
|
}
|
|
#if OMP_40_ENABLED
|
|
if( teams_swapped ) {
|
|
// Restore thread structure
|
|
th->th.th_info.ds.ds_tid = 0;
|
|
th->th.th_team = team;
|
|
th->th.th_team_nproc = team->t.t_nproc;
|
|
th->th.th_task_team = team->t.t_task_team[task_state];
|
|
th->th.th_task_state = task_state;
|
|
}
|
|
#endif
|
|
KA_TRACE( 10, ( "__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
|
|
|
|
return retval;
|
|
}
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid global thread id.
|
|
@param lck pointer to the unique lock data structure
|
|
|
|
Finish the execution of a reduce nowait.
|
|
*/
|
|
void
|
|
__kmpc_end_reduce_nowait( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ) {
|
|
|
|
PACKED_REDUCTION_METHOD_T packed_reduction_method;
|
|
|
|
KA_TRACE( 10, ( "__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
|
|
|
|
packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
|
|
|
|
if( packed_reduction_method == critical_reduce_block ) {
|
|
|
|
__kmp_end_critical_section_reduce_block( loc, global_tid, lck );
|
|
|
|
} else if( packed_reduction_method == empty_reduce_block ) {
|
|
|
|
// usage: if team size == 1, no synchronization is required ( on Intel platforms only )
|
|
|
|
} else if( packed_reduction_method == atomic_reduce_block ) {
|
|
|
|
// neither master nor other workers should get here
|
|
// (code gen does not generate this call in case 2: atomic reduce block)
|
|
// actually it's better to remove this elseif at all;
|
|
// after removal this value will checked by the 'else' and will assert
|
|
|
|
} else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
|
|
|
|
// only master gets here
|
|
|
|
} else {
|
|
|
|
// should never reach this block
|
|
KMP_ASSERT( 0 ); // "unexpected method"
|
|
|
|
}
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_sync( global_tid, ct_reduce, loc );
|
|
|
|
KA_TRACE( 10, ( "__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
|
|
|
|
return;
|
|
}
|
|
|
|
/* 2.a.ii. Reduce Block with a terminating barrier */
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid global thread number
|
|
@param num_vars number of items (variables) to be reduced
|
|
@param reduce_size size of data in bytes to be reduced
|
|
@param reduce_data pointer to data to be reduced
|
|
@param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data
|
|
@param lck pointer to the unique lock data structure
|
|
@result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed
|
|
|
|
A blocking reduce that includes an implicit barrier.
|
|
*/
|
|
kmp_int32
|
|
__kmpc_reduce(
|
|
ident_t *loc, kmp_int32 global_tid,
|
|
kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
|
|
void (*reduce_func)(void *lhs_data, void *rhs_data),
|
|
kmp_critical_name *lck )
|
|
{
|
|
KMP_COUNT_BLOCK(REDUCE_wait);
|
|
int retval = 0;
|
|
PACKED_REDUCTION_METHOD_T packed_reduction_method;
|
|
|
|
KA_TRACE( 10, ( "__kmpc_reduce() enter: called T#%d\n", global_tid ) );
|
|
|
|
// why do we need this initialization here at all?
|
|
// Reduction clause can not be a stand-alone directive.
|
|
|
|
// do not call __kmp_serial_initialize(), it will be called by __kmp_parallel_initialize() if needed
|
|
// possible detection of false-positive race by the threadchecker ???
|
|
if( ! TCR_4( __kmp_init_parallel ) )
|
|
__kmp_parallel_initialize();
|
|
|
|
// check correctness of reduce block nesting
|
|
#if KMP_USE_DYNAMIC_LOCK
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
|
|
#else
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_push_sync( global_tid, ct_reduce, loc, NULL );
|
|
#endif
|
|
|
|
packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
|
|
__KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
|
|
|
|
if( packed_reduction_method == critical_reduce_block ) {
|
|
|
|
__kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
|
|
retval = 1;
|
|
|
|
} else if( packed_reduction_method == empty_reduce_block ) {
|
|
|
|
// usage: if team size == 1, no synchronization is required ( Intel platforms only )
|
|
retval = 1;
|
|
|
|
} else if( packed_reduction_method == atomic_reduce_block ) {
|
|
|
|
retval = 2;
|
|
|
|
} else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
|
|
|
|
//case tree_reduce_block:
|
|
// this barrier should be visible to a customer and to the threading profile tool
|
|
// (it's a terminating barrier on constructs if NOWAIT not specified)
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc; // needed for correct notification of frames
|
|
#endif
|
|
retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
|
|
retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
|
|
|
|
// all other workers except master should do this pop here
|
|
// ( none of other workers except master will enter __kmpc_end_reduce() )
|
|
if ( __kmp_env_consistency_check ) {
|
|
if( retval == 0 ) { // 0: all other workers; 1: master
|
|
__kmp_pop_sync( global_tid, ct_reduce, loc );
|
|
}
|
|
}
|
|
|
|
} else {
|
|
|
|
// should never reach this block
|
|
KMP_ASSERT( 0 ); // "unexpected method"
|
|
|
|
}
|
|
|
|
KA_TRACE( 10, ( "__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
|
|
|
|
return retval;
|
|
}
|
|
|
|
/*!
|
|
@ingroup SYNCHRONIZATION
|
|
@param loc source location information
|
|
@param global_tid global thread id.
|
|
@param lck pointer to the unique lock data structure
|
|
|
|
Finish the execution of a blocking reduce.
|
|
The <tt>lck</tt> pointer must be the same as that used in the corresponding start function.
|
|
*/
|
|
void
|
|
__kmpc_end_reduce( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ) {
|
|
|
|
PACKED_REDUCTION_METHOD_T packed_reduction_method;
|
|
|
|
KA_TRACE( 10, ( "__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
|
|
|
|
packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
|
|
|
|
// this barrier should be visible to a customer and to the threading profile tool
|
|
// (it's a terminating barrier on constructs if NOWAIT not specified)
|
|
|
|
if( packed_reduction_method == critical_reduce_block ) {
|
|
|
|
__kmp_end_critical_section_reduce_block( loc, global_tid, lck );
|
|
|
|
// TODO: implicit barrier: should be exposed
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc;
|
|
#endif
|
|
__kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
|
|
|
|
} else if( packed_reduction_method == empty_reduce_block ) {
|
|
|
|
// usage: if team size == 1, no synchronization is required ( Intel platforms only )
|
|
|
|
// TODO: implicit barrier: should be exposed
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc;
|
|
#endif
|
|
__kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
|
|
|
|
} else if( packed_reduction_method == atomic_reduce_block ) {
|
|
|
|
// TODO: implicit barrier: should be exposed
|
|
#if USE_ITT_NOTIFY
|
|
__kmp_threads[global_tid]->th.th_ident = loc;
|
|
#endif
|
|
__kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
|
|
|
|
} else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
|
|
|
|
// only master executes here (master releases all other workers)
|
|
__kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
|
|
|
|
} else {
|
|
|
|
// should never reach this block
|
|
KMP_ASSERT( 0 ); // "unexpected method"
|
|
|
|
}
|
|
|
|
if ( __kmp_env_consistency_check )
|
|
__kmp_pop_sync( global_tid, ct_reduce, loc );
|
|
|
|
KA_TRACE( 10, ( "__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
|
|
|
|
return;
|
|
}
|
|
|
|
#undef __KMP_GET_REDUCTION_METHOD
|
|
#undef __KMP_SET_REDUCTION_METHOD
|
|
|
|
/*-- end of interface to fast scalable reduce routines ---------------------------------------------------------------*/
|
|
|
|
kmp_uint64
|
|
__kmpc_get_taskid() {
|
|
|
|
kmp_int32 gtid;
|
|
kmp_info_t * thread;
|
|
|
|
gtid = __kmp_get_gtid();
|
|
if ( gtid < 0 ) {
|
|
return 0;
|
|
}; // if
|
|
thread = __kmp_thread_from_gtid( gtid );
|
|
return thread->th.th_current_task->td_task_id;
|
|
|
|
} // __kmpc_get_taskid
|
|
|
|
|
|
kmp_uint64
|
|
__kmpc_get_parent_taskid() {
|
|
|
|
kmp_int32 gtid;
|
|
kmp_info_t * thread;
|
|
kmp_taskdata_t * parent_task;
|
|
|
|
gtid = __kmp_get_gtid();
|
|
if ( gtid < 0 ) {
|
|
return 0;
|
|
}; // if
|
|
thread = __kmp_thread_from_gtid( gtid );
|
|
parent_task = thread->th.th_current_task->td_parent;
|
|
return ( parent_task == NULL ? 0 : parent_task->td_task_id );
|
|
|
|
} // __kmpc_get_parent_taskid
|
|
|
|
void __kmpc_place_threads(int nS, int sO, int nC, int cO, int nT)
|
|
{
|
|
if ( ! __kmp_init_serial ) {
|
|
__kmp_serial_initialize();
|
|
}
|
|
__kmp_place_num_sockets = nS;
|
|
__kmp_place_socket_offset = sO;
|
|
__kmp_place_num_cores = nC;
|
|
__kmp_place_core_offset = cO;
|
|
__kmp_place_num_threads_per_core = nT;
|
|
}
|
|
|
|
#if OMP_45_ENABLED
|
|
/*!
|
|
@ingroup WORK_SHARING
|
|
@param loc source location information.
|
|
@param gtid global thread number.
|
|
@param num_dims number of associated doacross loops.
|
|
@param dims info on loops bounds.
|
|
|
|
Initialize doacross loop information.
|
|
Expect compiler send us inclusive bounds,
|
|
e.g. for(i=2;i<9;i+=2) lo=2, up=8, st=2.
|
|
*/
|
|
void
|
|
__kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, struct kmp_dim * dims)
|
|
{
|
|
int j, idx;
|
|
kmp_int64 last, trace_count;
|
|
kmp_info_t *th = __kmp_threads[gtid];
|
|
kmp_team_t *team = th->th.th_team;
|
|
kmp_uint32 *flags;
|
|
kmp_disp_t *pr_buf = th->th.th_dispatch;
|
|
dispatch_shared_info_t *sh_buf;
|
|
|
|
KA_TRACE(20,("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
|
|
gtid, num_dims, !team->t.t_serialized));
|
|
KMP_DEBUG_ASSERT(dims != NULL);
|
|
KMP_DEBUG_ASSERT(num_dims > 0);
|
|
|
|
if( team->t.t_serialized ) {
|
|
KA_TRACE(20,("__kmpc_doacross_init() exit: serialized team\n"));
|
|
return; // no dependencies if team is serialized
|
|
}
|
|
KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
|
|
idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for the next loop
|
|
sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
|
|
|
|
// Save bounds info into allocated private buffer
|
|
KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
|
|
pr_buf->th_doacross_info =
|
|
(kmp_int64*)__kmp_thread_malloc(th, sizeof(kmp_int64)*(4 * num_dims + 1));
|
|
KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
|
|
pr_buf->th_doacross_info[0] = (kmp_int64)num_dims; // first element is number of dimensions
|
|
// Save also address of num_done in order to access it later without knowing the buffer index
|
|
pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
|
|
pr_buf->th_doacross_info[2] = dims[0].lo;
|
|
pr_buf->th_doacross_info[3] = dims[0].up;
|
|
pr_buf->th_doacross_info[4] = dims[0].st;
|
|
last = 5;
|
|
for( j = 1; j < num_dims; ++j ) {
|
|
kmp_int64 range_length; // To keep ranges of all dimensions but the first dims[0]
|
|
if( dims[j].st == 1 ) { // most common case
|
|
// AC: should we care of ranges bigger than LLONG_MAX? (not for now)
|
|
range_length = dims[j].up - dims[j].lo + 1;
|
|
} else {
|
|
if( dims[j].st > 0 ) {
|
|
KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
|
|
range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
|
|
} else { // negative increment
|
|
KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
|
|
range_length = (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
|
|
}
|
|
}
|
|
pr_buf->th_doacross_info[last++] = range_length;
|
|
pr_buf->th_doacross_info[last++] = dims[j].lo;
|
|
pr_buf->th_doacross_info[last++] = dims[j].up;
|
|
pr_buf->th_doacross_info[last++] = dims[j].st;
|
|
}
|
|
|
|
// Compute total trip count.
|
|
// Start with range of dims[0] which we don't need to keep in the buffer.
|
|
if( dims[0].st == 1 ) { // most common case
|
|
trace_count = dims[0].up - dims[0].lo + 1;
|
|
} else if( dims[0].st > 0 ) {
|
|
KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
|
|
trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
|
|
} else { // negative increment
|
|
KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
|
|
trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
|
|
}
|
|
for( j = 1; j < num_dims; ++j ) {
|
|
trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges
|
|
}
|
|
KMP_DEBUG_ASSERT(trace_count > 0);
|
|
|
|
// Check if shared buffer is not occupied by other loop (idx - __kmp_dispatch_num_buffers)
|
|
if( idx != sh_buf->doacross_buf_idx ) {
|
|
// Shared buffer is occupied, wait for it to be free
|
|
__kmp_wait_yield_4( (kmp_uint32*)&sh_buf->doacross_buf_idx, idx, __kmp_eq_4, NULL );
|
|
}
|
|
// Check if we are the first thread. After the CAS the first thread gets 0,
|
|
// others get 1 if initialization is in progress, allocated pointer otherwise.
|
|
flags = (kmp_uint32*)KMP_COMPARE_AND_STORE_RET64(
|
|
(kmp_int64*)&sh_buf->doacross_flags,NULL,(kmp_int64)1);
|
|
if( flags == NULL ) {
|
|
// we are the first thread, allocate the array of flags
|
|
kmp_int64 size = trace_count / 8 + 8; // in bytes, use single bit per iteration
|
|
sh_buf->doacross_flags = (kmp_uint32*)__kmp_thread_calloc(th, size, 1);
|
|
} else if( (kmp_int64)flags == 1 ) {
|
|
// initialization is still in progress, need to wait
|
|
while( (volatile kmp_int64)sh_buf->doacross_flags == 1 ) {
|
|
KMP_YIELD(TRUE);
|
|
}
|
|
}
|
|
KMP_DEBUG_ASSERT((kmp_int64)sh_buf->doacross_flags > 1); // check value of pointer
|
|
pr_buf->th_doacross_flags = sh_buf->doacross_flags; // save private copy in order to not
|
|
// touch shared buffer on each iteration
|
|
KA_TRACE(20,("__kmpc_doacross_init() exit: T#%d\n", gtid));
|
|
}
|
|
|
|
void
|
|
__kmpc_doacross_wait(ident_t *loc, int gtid, long long *vec)
|
|
{
|
|
kmp_int32 shft, num_dims, i;
|
|
kmp_uint32 flag;
|
|
kmp_int64 iter_number; // iteration number of "collapsed" loop nest
|
|
kmp_info_t *th = __kmp_threads[gtid];
|
|
kmp_team_t *team = th->th.th_team;
|
|
kmp_disp_t *pr_buf;
|
|
kmp_int64 lo, up, st;
|
|
|
|
KA_TRACE(20,("__kmpc_doacross_wait() enter: called T#%d\n", gtid));
|
|
if( team->t.t_serialized ) {
|
|
KA_TRACE(20,("__kmpc_doacross_wait() exit: serialized team\n"));
|
|
return; // no dependencies if team is serialized
|
|
}
|
|
|
|
// calculate sequential iteration number and check out-of-bounds condition
|
|
pr_buf = th->th.th_dispatch;
|
|
KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
|
|
num_dims = pr_buf->th_doacross_info[0];
|
|
lo = pr_buf->th_doacross_info[2];
|
|
up = pr_buf->th_doacross_info[3];
|
|
st = pr_buf->th_doacross_info[4];
|
|
if( st == 1 ) { // most common case
|
|
if( vec[0] < lo || vec[0] > up ) {
|
|
KA_TRACE(20,(
|
|
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
|
|
gtid, vec[0], lo, up));
|
|
return;
|
|
}
|
|
iter_number = vec[0] - lo;
|
|
} else if( st > 0 ) {
|
|
if( vec[0] < lo || vec[0] > up ) {
|
|
KA_TRACE(20,(
|
|
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
|
|
gtid, vec[0], lo, up));
|
|
return;
|
|
}
|
|
iter_number = (kmp_uint64)(vec[0] - lo) / st;
|
|
} else { // negative increment
|
|
if( vec[0] > lo || vec[0] < up ) {
|
|
KA_TRACE(20,(
|
|
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
|
|
gtid, vec[0], lo, up));
|
|
return;
|
|
}
|
|
iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
|
|
}
|
|
for( i = 1; i < num_dims; ++i ) {
|
|
kmp_int64 iter, ln;
|
|
kmp_int32 j = i * 4;
|
|
ln = pr_buf->th_doacross_info[j + 1];
|
|
lo = pr_buf->th_doacross_info[j + 2];
|
|
up = pr_buf->th_doacross_info[j + 3];
|
|
st = pr_buf->th_doacross_info[j + 4];
|
|
if( st == 1 ) {
|
|
if( vec[i] < lo || vec[i] > up ) {
|
|
KA_TRACE(20,(
|
|
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
|
|
gtid, vec[i], lo, up));
|
|
return;
|
|
}
|
|
iter = vec[i] - lo;
|
|
} else if( st > 0 ) {
|
|
if( vec[i] < lo || vec[i] > up ) {
|
|
KA_TRACE(20,(
|
|
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
|
|
gtid, vec[i], lo, up));
|
|
return;
|
|
}
|
|
iter = (kmp_uint64)(vec[i] - lo) / st;
|
|
} else { // st < 0
|
|
if( vec[i] > lo || vec[i] < up ) {
|
|
KA_TRACE(20,(
|
|
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
|
|
gtid, vec[i], lo, up));
|
|
return;
|
|
}
|
|
iter = (kmp_uint64)(lo - vec[i]) / (-st);
|
|
}
|
|
iter_number = iter + ln * iter_number;
|
|
}
|
|
shft = iter_number % 32; // use 32-bit granularity
|
|
iter_number >>= 5; // divided by 32
|
|
flag = 1 << shft;
|
|
while( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 ) {
|
|
KMP_YIELD(TRUE);
|
|
}
|
|
KA_TRACE(20,("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
|
|
gtid, (iter_number<<5)+shft));
|
|
}
|
|
|
|
void
|
|
__kmpc_doacross_post(ident_t *loc, int gtid, long long *vec)
|
|
{
|
|
kmp_int32 shft, num_dims, i;
|
|
kmp_uint32 flag;
|
|
kmp_int64 iter_number; // iteration number of "collapsed" loop nest
|
|
kmp_info_t *th = __kmp_threads[gtid];
|
|
kmp_team_t *team = th->th.th_team;
|
|
kmp_disp_t *pr_buf;
|
|
kmp_int64 lo, st;
|
|
|
|
KA_TRACE(20,("__kmpc_doacross_post() enter: called T#%d\n", gtid));
|
|
if( team->t.t_serialized ) {
|
|
KA_TRACE(20,("__kmpc_doacross_post() exit: serialized team\n"));
|
|
return; // no dependencies if team is serialized
|
|
}
|
|
|
|
// calculate sequential iteration number (same as in "wait" but no out-of-bounds checks)
|
|
pr_buf = th->th.th_dispatch;
|
|
KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
|
|
num_dims = pr_buf->th_doacross_info[0];
|
|
lo = pr_buf->th_doacross_info[2];
|
|
st = pr_buf->th_doacross_info[4];
|
|
if( st == 1 ) { // most common case
|
|
iter_number = vec[0] - lo;
|
|
} else if( st > 0 ) {
|
|
iter_number = (kmp_uint64)(vec[0] - lo) / st;
|
|
} else { // negative increment
|
|
iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
|
|
}
|
|
for( i = 1; i < num_dims; ++i ) {
|
|
kmp_int64 iter, ln;
|
|
kmp_int32 j = i * 4;
|
|
ln = pr_buf->th_doacross_info[j + 1];
|
|
lo = pr_buf->th_doacross_info[j + 2];
|
|
st = pr_buf->th_doacross_info[j + 4];
|
|
if( st == 1 ) {
|
|
iter = vec[i] - lo;
|
|
} else if( st > 0 ) {
|
|
iter = (kmp_uint64)(vec[i] - lo) / st;
|
|
} else { // st < 0
|
|
iter = (kmp_uint64)(lo - vec[i]) / (-st);
|
|
}
|
|
iter_number = iter + ln * iter_number;
|
|
}
|
|
shft = iter_number % 32; // use 32-bit granularity
|
|
iter_number >>= 5; // divided by 32
|
|
flag = 1 << shft;
|
|
if( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 )
|
|
KMP_TEST_THEN_OR32( (kmp_int32*)&pr_buf->th_doacross_flags[iter_number], (kmp_int32)flag );
|
|
KA_TRACE(20,("__kmpc_doacross_post() exit: T#%d iter %lld posted\n",
|
|
gtid, (iter_number<<5)+shft));
|
|
}
|
|
|
|
void
|
|
__kmpc_doacross_fini(ident_t *loc, int gtid)
|
|
{
|
|
kmp_int64 num_done;
|
|
kmp_info_t *th = __kmp_threads[gtid];
|
|
kmp_team_t *team = th->th.th_team;
|
|
kmp_disp_t *pr_buf = th->th.th_dispatch;
|
|
|
|
KA_TRACE(20,("__kmpc_doacross_fini() enter: called T#%d\n", gtid));
|
|
if( team->t.t_serialized ) {
|
|
KA_TRACE(20,("__kmpc_doacross_fini() exit: serialized team %p\n", team));
|
|
return; // nothing to do
|
|
}
|
|
num_done = KMP_TEST_THEN_INC64((kmp_int64*)pr_buf->th_doacross_info[1]) + 1;
|
|
if( num_done == th->th.th_team_nproc ) {
|
|
// we are the last thread, need to free shared resources
|
|
int idx = pr_buf->th_doacross_buf_idx - 1;
|
|
dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
|
|
KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] == (kmp_int64)&sh_buf->doacross_num_done);
|
|
KMP_DEBUG_ASSERT(num_done == (kmp_int64)sh_buf->doacross_num_done);
|
|
KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
|
|
__kmp_thread_free(th, (void*)sh_buf->doacross_flags);
|
|
sh_buf->doacross_flags = NULL;
|
|
sh_buf->doacross_num_done = 0;
|
|
sh_buf->doacross_buf_idx += __kmp_dispatch_num_buffers; // free buffer for future re-use
|
|
}
|
|
// free private resources (need to keep buffer index forever)
|
|
__kmp_thread_free(th, (void*)pr_buf->th_doacross_info);
|
|
pr_buf->th_doacross_info = NULL;
|
|
KA_TRACE(20,("__kmpc_doacross_fini() exit: T#%d\n", gtid));
|
|
}
|
|
#endif
|
|
|
|
// end of file //
|
|
|