ESPResSo
Extensible Simulation Package for Research on Soft Matter Systems
Loading...
Searching...
No Matches
init.cpp
Go to the documentation of this file.
1/*
2 * Copyright (C) 2010-2022 The ESPResSo project
3 *
4 * This file is part of ESPResSo.
5 *
6 * ESPResSo is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * ESPResSo is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include "config/config.hpp"
21
22#ifdef CUDA
23
24#include "init.hpp"
25#include "utils.hpp"
26
27#include "communication.hpp"
28
29#include <mpi.h>
30
31#include <cstring>
32#include <iterator>
33#include <set>
34#include <vector>
35
36/** Helper class for device sets.
37 */
40 const EspressoGpuDevice &b) const {
41 auto const name_comp = strncmp(a.proc_name, b.proc_name, 63);
42 /* if both devices are from the same node, order by id */
43 return (name_comp == 0) ? a.id < b.id : name_comp < 0;
44 }
45};
46
47/** Gather list of CUDA devices on all nodes on the head node.
48 * It relies on <tt>MPI_Get_processor_name()</tt> to get a unique identifier
49 * of the physical node, as opposed to the logical rank of which there can
50 * be more than one per node.
51 */
52std::vector<EspressoGpuDevice> cuda_gather_gpus() {
53 /* List of local devices */
54 std::vector<EspressoGpuDevice> devices_local;
55 /* Global unique device list (only relevant on the head node) */
56 std::vector<EspressoGpuDevice> devices_global;
57
58 int n_devices = 0;
60 [&n_devices]() { n_devices = cuda_get_n_gpus(); });
61
62 int proc_name_len;
63 char proc_name[MPI_MAX_PROCESSOR_NAME];
64 MPI_Get_processor_name(proc_name, &proc_name_len);
65 proc_name[63] = '\0';
66
67 invoke_skip_cuda_exceptions([&devices_local, n_devices, &proc_name]() {
68 for (int i = 0; i < n_devices; ++i) {
69 auto device = cuda_get_device_props(i);
70 std::strncpy(device.proc_name, proc_name, 64);
71 device.proc_name[63] = '\0';
72 device.node = this_node;
73 devices_local.emplace_back(device);
74 }
75 });
76
77 auto const n_gpus = static_cast<int>(devices_local.size());
78 auto const n_nodes = ::communicator.size;
79
80 if (this_node == 0) {
81 std::set<EspressoGpuDevice, CompareDevices> device_set;
82 int *n_gpu_array = new int[static_cast<unsigned int>(n_nodes)];
83 MPI_Gather(&n_gpus, 1, MPI_INT, n_gpu_array, 1, MPI_INT, 0, MPI_COMM_WORLD);
84
85 /* insert local devices */
86 std::copy(devices_local.begin(), devices_local.end(),
87 std::inserter(device_set, device_set.begin()));
88
89 EspressoGpuDevice device;
90 MPI_Status s;
91 /* Get devices from other nodes */
92 for (int i = 1; i < n_nodes; ++i) {
93 for (int j = 0; j < n_gpu_array[i]; ++j) {
94 MPI_Recv(&device, sizeof(EspressoGpuDevice), MPI_BYTE, i, 0,
95 MPI_COMM_WORLD, &s);
96 device_set.insert(device);
97 }
98 }
99 /* Copy unique devices to result, if any */
100 std::copy(device_set.begin(), device_set.end(),
101 std::inserter(devices_global, devices_global.begin()));
102 delete[] n_gpu_array;
103 } else {
104 /* Send number of devices to head node */
105 MPI_Gather(&n_gpus, 1, MPI_INT, nullptr, 1, MPI_INT, 0, MPI_COMM_WORLD);
106 /* Send devices to head node */
107 for (auto const &device : devices_local) {
108 MPI_Send(&device, sizeof(EspressoGpuDevice), MPI_BYTE, 0, 0,
109 MPI_COMM_WORLD);
110 }
111 }
112 return devices_global;
113}
114
120
121#endif // CUDA
Communicator communicator
int this_node
The number of this node.
This file contains the defaults for ESPResSo.
void invoke_skip_cuda_exceptions(F &&f, Args &&...args)
Invoke a function and silently ignore any thrown cuda_runtime_error error.
std::vector< EspressoGpuDevice > cuda_gather_gpus()
Gather list of CUDA devices on all nodes on the head node.
Definition init.cpp:52
void cuda_on_program_start()
Called on program start.
Definition init.cpp:115
EspressoGpuDevice cuda_get_device_props(int dev)
Get properties of a CUDA device.
Definition init_cuda.cu:73
int cuda_get_n_gpus()
Get the number of CUDA devices.
Definition init_cuda.cu:43
void cuda_init()
Initializes the CUDA stream.
Definition init_cuda.cu:41
int size
The MPI world size.
Helper class for device sets.
Definition init.cpp:38
bool operator()(const EspressoGpuDevice &a, const EspressoGpuDevice &b) const
Definition init.cpp:39
Struct to hold information relevant to ESPResSo about GPUs.
Definition init.hpp:35
char proc_name[64]
Node identification.
Definition init.hpp:41
int id
Local CUDA device id.
Definition init.hpp:37