class nbla::MultiProcessDataParallelCommunicator

template<typename T>
class MultiProcessDataParallelCommunicator : public nbla::Communicator

MultiProcessDataParallelCommunicator interface which is extended to implement a new DataParallelcommunicator class.

MultiProcessDataParallelCommunicator exchanges gradients parameters or parameters itself.

Public Functions

MultiProcessDataParallelCommunicator(const Context &ctx)

Constructor takes at least context and parameters.

Parameters:

ctx – Context

void add_context_and_parameters(const pair<Context, vector<pair<string, VariablePtr>>> &ctx_params)

Adding context and parameters communicated via this class.

Parameters:

cparams – pair<Context, vector of pair<name, VariablePtr>

void remove_context_parameters(const pair<Context, vector<string>> &ctx_keys)

Remove previously registered parameters by keys.

void clear_context_parameters()

Clear all parameters.

virtual void init()

Initall or initrank, depending multi-threads or multi-processes.

This function MUST be called after all parameters communicated are added by `add_context_and_parameters` method.

virtual string new_group(pair<string, vector<int>> name_ranks_pair)

Create group.

virtual unordered_map<string, vector<int>> list_groups()

List groups.

virtual vector<int> find_group(const string &group)

Find groups.

virtual void reduce(const vector<NdArrayPtr> &ndarray_list, int dst, bool division = false, bool inplace = false, const string &group = "world")

reduce over parameters added.

Parameters:

  • ndarray_list – Vector of NdArrayPtr.

  • dst – Destination rank.

  • division – Divide the reduced value.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void reduce(NdArrayPtr ndarray, int dst, bool division = false, bool inplace = false, const string &group = "world")

reduce over parameters added.

Parameters:

  • data – NdArrayPtr.

  • dst – Destination rank.

  • division – Divide the reduced value.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void allreduce(bool division = false, bool inplace = false)

allreduce over parameters added.

Deprecated. Use all_reduce.

Currently, allreduce is applied to gradient regions.

Parameters:

  • division – Divide the reduced value.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void all_reduce(const vector<NdArrayPtr> &ndarray_list, bool division = false, bool inplace = false, const string &group = "world")

all_reduce over parameters added.

Parameters:

  • ndarray_list – Vector of NdArrayPtr

  • division – Divide the reduced value.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void all_reduce(NdArrayPtr ndarray, bool division = false, bool inplace = false, const string &group = "world")

all_reduce over parameters added.

Parameters:

  • data – NdArrayPtr

  • division – Divide the reduced value.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void reduce_scatter(const vector<NdArrayPtr> &ndarray_list, NdArrayPtr ndarray, bool division = false, const string &group = "world")

reducescatter.

Parameters:

  • ndarray_list – Vector of NdArrayPtr

  • ndarray – NdArrayPtr

  • division – Divide the reduced value.

  • group – Name of a group.

virtual void bcast(const vector<NdArrayPtr> &ndarray_list, int src, bool inplace = false, const string &group = "world")

broadcast.

Parameters:

  • ndarray_list – Vector of NdArrayPtr.

  • src – Source rank.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void bcast(NdArrayPtr ndarray, int src, bool inplace = false, const string &group = "world")

broadcast.

Parameters:

  • data – NdArrayPtr.

  • src – Source rank.

  • inplace – Pack the arrays into one large array if false.

  • group – Name of a group.

virtual void all_gather(NdArrayPtr ndarray, const vector<NdArrayPtr> &ndarray_list, const string &group = "world")

all_gather.

Parameters:

  • ndarray – data to be sent.

  • ndarray_list – Vector of NdArrayPtr to receive data.

  • group – Name of a group.

virtual void reduce_async(bool division = false)

reduce asynchronously.

Parameters:

division – Divide the reduced value.

virtual void allreduce_async(bool division = false, bool inplace = false)

reduce asynchronously.

Parameters:

  • division – Divide the reduced value.

  • inplace – Pack the arrays into one large array if false.

virtual void reducescatter_async(bool division = false)

reducescatter asynchronously.

Parameters:

division – Divide the reduced value.

virtual void bcast_async()

broadcast asynchronously.

virtual void allgather_async()

allgather asynchronously.

vector<string> allowed_array_classes()

Get array classes that are allowed to be specified by Context.