StarPU Handbook
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Groups Pages
MPI Support

Initialisation

#define STARPU_USE_MPI
 
int starpu_mpi_init (int *argc, char ***argv, int initialize_mpi)
 
int starpu_mpi_initialize (void)
 
int starpu_mpi_initialize_extended (int *rank, int *world_size)
 
int starpu_mpi_shutdown (void)
 
void starpu_mpi_comm_amounts_retrieve (size_t *comm_amounts)
 

Communication

int starpu_mpi_send (starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm)
 
int starpu_mpi_recv (starpu_data_handle_t data_handle, int source, int mpi_tag, MPI_Comm comm, MPI_Status *status)
 
int starpu_mpi_isend (starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int mpi_tag, MPI_Comm comm)
 
int starpu_mpi_irecv (starpu_data_handle_t data_handle, starpu_mpi_req *req, int source, int mpi_tag, MPI_Comm comm)
 
int starpu_mpi_isend_detached (starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm, void(*callback)(void *), void *arg)
 
int starpu_mpi_irecv_detached (starpu_data_handle_t data_handle, int source, int mpi_tag, MPI_Comm comm, void(*callback)(void *), void *arg)
 
int starpu_mpi_wait (starpu_mpi_req *req, MPI_Status *status)
 
int starpu_mpi_test (starpu_mpi_req *req, int *flag, MPI_Status *status)
 
int starpu_mpi_barrier (MPI_Comm comm)
 
int starpu_mpi_isend_detached_unlock_tag (starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm, starpu_tag_t tag)
 
int starpu_mpi_irecv_detached_unlock_tag (starpu_data_handle_t data_handle, int source, int mpi_tag, MPI_Comm comm, starpu_tag_t tag)
 
int starpu_mpi_isend_array_detached_unlock_tag (unsigned array_size, starpu_data_handle_t *data_handle, int *dest, int *mpi_tag, MPI_Comm *comm, starpu_tag_t tag)
 
int starpu_mpi_irecv_array_detached_unlock_tag (unsigned array_size, starpu_data_handle_t *data_handle, int *source, int *mpi_tag, MPI_Comm *comm, starpu_tag_t tag)
 

Communication Cache

void starpu_mpi_cache_flush (MPI_Comm comm, starpu_data_handle_t data_handle)
 
void starpu_mpi_cache_flush_all_data (MPI_Comm comm)
 

MPI Insert Task

#define STARPU_EXECUTE_ON_NODE
 
#define STARPU_EXECUTE_ON_DATA
 
int starpu_data_set_tag (starpu_data_handle_t handle, int tag)
 
int starpu_data_get_tag (starpu_data_handle_t handle)
 
void starpu_mpi_data_register (starpu_data_handle_t data_handle, int tag, int rank)
 
int starpu_data_set_rank (starpu_data_handle_t handle, int rank)
 
int starpu_data_get_rank (starpu_data_handle_t handle)
 
int starpu_mpi_insert_task (MPI_Comm comm, struct starpu_codelet *codelet,...)
 
void starpu_mpi_get_data_on_node (MPI_Comm comm, starpu_data_handle_t data_handle, int node)
 
void starpu_mpi_get_data_on_node_detached (MPI_Comm comm, starpu_data_handle_t data_handle, int node, void(*callback)(void *), void *arg)
 

Collective Operations

void starpu_mpi_redux_data (MPI_Comm comm, starpu_data_handle_t data_handle)
 
int starpu_mpi_scatter_detached (starpu_data_handle_t *data_handles, int count, int root, MPI_Comm comm, void(*scallback)(void *), void *sarg, void(*rcallback)(void *), void *rarg)
 
int starpu_mpi_gather_detached (starpu_data_handle_t *data_handles, int count, int root, MPI_Comm comm, void(*scallback)(void *), void *sarg, void(*rcallback)(void *), void *rarg)
 

Detailed Description

Macro Definition Documentation

#define STARPU_USE_MPI

This macro is defined when StarPU has been installed with MPI support. It should be used in your code to detect the availability of MPI.

#define STARPU_EXECUTE_ON_NODE

this macro is used when calling starpu_mpi_insert_task(), and must be followed by a integer value which specified the node on which to execute the codelet.

#define STARPU_EXECUTE_ON_DATA

this macro is used when calling starpu_mpi_insert_task(), and must be followed by a data handle to specify that the node owning the given data will execute the codelet.

Function Documentation

int starpu_mpi_init ( int *  argc,
char ***  argv,
int  initialize_mpi 
)

Initializes the starpumpi library. initialize_mpi indicates if MPI should be initialized or not by StarPU. If the value is not 0, MPI will be initialized by calling MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...).

int starpu_mpi_initialize ( void  )
Deprecated:
This function has been made deprecated. One should use instead the function starpu_mpi_init(). This function does not call MPI_Init(), it should be called beforehand.
int starpu_mpi_initialize_extended ( int *  rank,
int *  world_size 
)
Deprecated:
This function has been made deprecated. One should use instead the function starpu_mpi_init(). MPI will be initialized by starpumpi by calling MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...).
int starpu_mpi_shutdown ( void  )

Cleans the starpumpi library. This must be called between calling starpu_mpi functions and starpu_shutdown(). MPI_Finalize() will be called if StarPU-MPI has been initialized by starpu_mpi_init().

void starpu_mpi_comm_amounts_retrieve ( size_t *  comm_amounts)

Retrieve the current amount of communications from the current node in the array comm_amounts which must have a size greater or equal to the world size. Communications statistics must be enabled (see STARPU_COMM_STATS).

int starpu_mpi_send ( starpu_data_handle_t  data_handle,
int  dest,
int  mpi_tag,
MPI_Comm  comm 
)

Performs a standard-mode, blocking send of data_handle to the node dest using the message tag mpi_tag within the communicator comm.

int starpu_mpi_recv ( starpu_data_handle_t  data_handle,
int  source,
int  mpi_tag,
MPI_Comm  comm,
MPI_Status *  status 
)

Performs a standard-mode, blocking receive in data_handle from the node source using the message tag mpi_tag within the communicator comm.

int starpu_mpi_isend ( starpu_data_handle_t  data_handle,
starpu_mpi_req *  req,
int  dest,
int  mpi_tag,
MPI_Comm  comm 
)

Posts a standard-mode, non blocking send of data_handle to the node dest using the message tag mpi_tag within the communicator comm. After the call, the pointer to the request req can be used to test or to wait for the completion of the communication.

int starpu_mpi_irecv ( starpu_data_handle_t  data_handle,
starpu_mpi_req *  req,
int  source,
int  mpi_tag,
MPI_Comm  comm 
)

Posts a nonblocking receive in data_handle from the node source using the message tag mpi_tag within the communicator comm. After the call, the pointer to the request req can be used to test or to wait for the completion of the communication.

int starpu_mpi_isend_detached ( starpu_data_handle_t  data_handle,
int  dest,
int  mpi_tag,
MPI_Comm  comm,
void(*)(void *)  callback,
void *  arg 
)

Posts a standard-mode, non blocking send of data_handle to the node dest using the message tag mpi_tag within the communicator comm. On completion, the callback function is called with the argument arg. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request.

int starpu_mpi_irecv_detached ( starpu_data_handle_t  data_handle,
int  source,
int  mpi_tag,
MPI_Comm  comm,
void(*)(void *)  callback,
void *  arg 
)

Posts a nonblocking receive in data_handle from the node source using the message tag mpi_tag within the communicator comm. On completion, the callback function is called with the argument arg. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request.

int starpu_mpi_wait ( starpu_mpi_req *  req,
MPI_Status *  status 
)

Returns when the operation identified by request req is complete.

int starpu_mpi_test ( starpu_mpi_req *  req,
int *  flag,
MPI_Status *  status 
)

If the operation identified by req is complete, set flag to 1. The status object is set to contain information on the completed operation.

int starpu_mpi_barrier ( MPI_Comm  comm)

Blocks the caller until all group members of the communicator comm have called it.

int starpu_mpi_isend_detached_unlock_tag ( starpu_data_handle_t  data_handle,
int  dest,
int  mpi_tag,
MPI_Comm  comm,
starpu_tag_t  tag 
)

Posts a standard-mode, non blocking send of data_handle to the node dest using the message tag mpi_tag within the communicator comm. On completion, tag is unlocked.

int starpu_mpi_irecv_detached_unlock_tag ( starpu_data_handle_t  data_handle,
int  source,
int  mpi_tag,
MPI_Comm  comm,
starpu_tag_t  tag 
)

Posts a nonblocking receive in data_handle from the node source using the message tag mpi_tag within the communicator comm. On completion, tag is unlocked.

int starpu_mpi_isend_array_detached_unlock_tag ( unsigned  array_size,
starpu_data_handle_t data_handle,
int *  dest,
int *  mpi_tag,
MPI_Comm *  comm,
starpu_tag_t  tag 
)

Posts array_size standard-mode, non blocking send. Each post sends the n-th data of the array data_handle to the n-th node of the array dest using the n-th message tag of the array mpi_tag within the n-th communicator of the array comm. On completion of the all the requests, tag is unlocked.

int starpu_mpi_irecv_array_detached_unlock_tag ( unsigned  array_size,
starpu_data_handle_t data_handle,
int *  source,
int *  mpi_tag,
MPI_Comm *  comm,
starpu_tag_t  tag 
)

Posts array_size nonblocking receive. Each post receives in the n-th data of the array data_handle from the n-th node of the array source using the n-th message tag of the array mpi_tag within the n-th communicator of the array comm. On completion of the all the requests, tag is unlocked.

void starpu_mpi_cache_flush ( MPI_Comm  comm,
starpu_data_handle_t  data_handle 
)

Clear the send and receive communication cache for the data data_handle and invalidate the value. The function has to be called synchronously by all the MPI nodes. The function does nothing if the cache mechanism is disabled (see STARPU_MPI_CACHE).

void starpu_mpi_cache_flush_all_data ( MPI_Comm  comm)

Clear the send and receive communication cache for all data and invalidate their values. The function has to be called synchronously by all the MPI nodes. The function does nothing if the cache mechanism is disabled (see STARPU_MPI_CACHE).

int starpu_data_set_tag ( starpu_data_handle_t  handle,
int  tag 
)

Tell StarPU-MPI which MPI tag to use when exchanging the data.

int starpu_data_get_tag ( starpu_data_handle_t  handle)

Returns the MPI tag to be used when exchanging the data.

int starpu_mpi_data_register ( starpu_data_handle_t  handle,
int  tag,
int  rank 
)

Calling this function should be prefered to calling both starpu_data_set_rank() and starpu_data_set_tag() as it also allows to automatically clear the MPI communication cache when unregistering the data.

int starpu_data_set_rank ( starpu_data_handle_t  handle,
int  rank 
)

Tell StarPU-MPI which MPI node "owns" a given data, that is, the node which will always keep an up-to-date value, and will by default execute tasks which write to it.

int starpu_data_get_rank ( starpu_data_handle_t  handle)

Returns the last value set by starpu_data_set_rank().

int starpu_mpi_insert_task ( MPI_Comm  comm,
struct starpu_codelet codelet,
  ... 
)

Create and submit a task corresponding to codelet with the following arguments. The argument list must be zero-terminated.

The arguments following the codelets are the same types as for the function starpu_insert_task(). The extra argument STARPU_EXECUTE_ON_NODE followed by an integer allows to specify the MPI node to execute the codelet. It is also possible to specify that the node owning a specific data will execute the codelet, by using STARPU_EXECUTE_ON_DATA followed by a data handle.

The internal algorithm is as follows:

  1. Find out which MPI node is going to execute the codelet.
    • If there is only one node owning data in STARPU_W mode, it will be selected;
    • If there is several nodes owning data in STARPU_W node, the one selected will be the one having the least data in R mode so as to minimize the amount of data to be transfered;
    • The argument STARPU_EXECUTE_ON_NODE followed by an integer can be used to specify the node;
    • The argument STARPU_EXECUTE_ON_DATA followed by a data handle can be used to specify that the node owing the given data will execute the codelet.
  2. Send and receive data as requested. Nodes owning data which need to be read by the task are sending them to the MPI node which will execute it. The latter receives them.
  3. Execute the codelet. This is done by the MPI node selected in the 1st step of the algorithm.
  4. If several MPI nodes own data to be written to, send written data back to their owners.

The algorithm also includes a communication cache mechanism that allows not to send data twice to the same MPI node, unless the data has been modified. The cache can be disabled (see STARPU_MPI_CACHE).

void starpu_mpi_get_data_on_node ( MPI_Comm  comm,
starpu_data_handle_t  data_handle,
int  node 
)

Transfer data data_handle to MPI node node, sending it from its owner if needed. At least the target node and the owner have to call the function.

void starpu_mpi_get_data_on_node_detached ( MPI_Comm  comm,
starpu_data_handle_t  data_handle,
int  node,
void(*)(void *)  callback,
void *  arg 
)

Transfer data data_handle to MPI node node, sending it from its owner if needed. At least the target node and the owner have to call the function. On reception, the callback function is called with the argument arg.

void starpu_mpi_redux_data ( MPI_Comm  comm,
starpu_data_handle_t  data_handle 
)

Perform a reduction on the given data. All nodes send the data to its owner node which will perform a reduction.

int starpu_mpi_scatter_detached ( starpu_data_handle_t data_handles,
int  count,
int  root,
MPI_Comm  comm,
void(*)(void *)  scallback,
void *  sarg,
void(*)(void *)  rcallback,
void *  rarg 
)

Scatter data among processes of the communicator based on the ownership of the data. For each data of the array data_handles, the process root sends the data to the process owning this data. Processes receiving data must have valid data handles to receive them. On completion of the collective communication, the scallback function is called with the argument sarg on the process root, the rcallback function is called with the argument rarg on any other process.

int starpu_mpi_gather_detached ( starpu_data_handle_t data_handles,
int  count,
int  root,
MPI_Comm  comm,
void(*)(void *)  scallback,
void *  sarg,
void(*)(void *)  rcallback,
void *  rarg 
)

Gather data from the different processes of the communicator onto the process root. Each process owning data handle in the array data_handles will send them to the process root. The process root must have valid data handles to receive the data. On completion of the collective communication, the rcallback function is called with the argument rarg on the process root, the scallback function is called with the argument sarg on any other process.