57#include "system/System.hpp"
61#include <boost/archive/binary_iarchive.hpp>
62#include <boost/archive/binary_oarchive.hpp>
63#include <boost/iostreams/device/array.hpp>
64#include <boost/iostreams/device/back_inserter.hpp>
65#include <boost/iostreams/stream.hpp>
92static bool fatal_error(
char const *msg, std::string
const &fn =
"",
93 std::string
const &extra =
"") {
94 std::stringstream what;
95 what <<
"MPI-IO Error: " << msg;
97 what <<
" \"" << fn <<
"\"";
99 if (not extra.empty()) {
100 what <<
" :" << extra;
103 MPI_Comm_size(MPI_COMM_WORLD, &size);
105 throw std::runtime_error(what.str());
107 fprintf(stderr,
"%s\n", what.str().c_str());
122static bool fatal_error(
char const *msg, std::string
const &fn, MPI_File *fp,
125 char buf[MPI_MAX_ERROR_STRING];
127 MPI_Error_string(errnum, buf, &buf_len);
149 std::size_t len, std::size_t pref,
150 MPI_Datatype MPI_T) {
153 ret = MPI_File_open(MPI_COMM_WORLD,
const_cast<char *
>(fn.c_str()),
155 MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_EXCL,
161 static_cast<MPI_Offset
>(pref) *
static_cast<MPI_Offset
>(
sizeof(T));
162 ret = MPI_File_set_view(f, offset, MPI_T, MPI_T,
const_cast<char *
>(
"native"),
164 ret |= MPI_File_write_all(f, arr,
static_cast<int>(len), MPI_T,
166 static_cast<void>(ret and
fatal_error(
"Could not write file", fn, &f, ret));
176 unsigned long offset = 0ul;
177 MPI_Exscan(&n_items, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD);
189static void dump_info(std::string
const &fn,
unsigned fields,
192 auto const nbonds = bonded_ias.
size();
193 assert(
static_cast<std::size_t
>(bonded_ias.
get_next_key()) == nbonds);
195 FILE *f = fopen(fn.c_str(),
"wb");
199 std::vector<int> npartners;
200 bool success = (fwrite(&fields,
sizeof(fields), 1u, f) == 1);
202 npartners.reserve(nbonds);
203 for (
int bond_id = 0; bond_id < bonded_ias.
get_next_key(); ++bond_id) {
208 success &= fwrite(&nbonds,
sizeof(std::size_t), 1u, f) == 1;
209 success &= fwrite(npartners.data(),
sizeof(
int), nbonds, f) == nbonds;
211 static_cast<void>(success or
fatal_error(
"Could not write file", fn));
218 auto const nlocalpart =
static_cast<unsigned long>(particles.
size());
221 auto &pos = buffers.
pos;
222 auto &vel = buffers.
vel;
223 auto &
id = buffers.
id;
224 auto &type = buffers.
type;
227 if (nlocalpart >
id.size())
228 id.resize(nlocalpart);
230 pos.resize(3ul * nlocalpart);
232 vel.resize(3ul * nlocalpart);
234 type.resize(nlocalpart);
237 auto id_it =
id.begin();
238 auto type_it = type.begin();
239 auto pos_it = pos.begin();
240 auto vel_it = vel.begin();
241 for (
auto const &p : particles) {
245 std::copy_n(std::begin(p.pos()), 3u, pos_it);
249 std::copy_n(std::begin(p.v()), 3u, vel_it);
259 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
261 dump_info(prefix +
".head", fields, bonded_ias);
262 auto const pref_offset =
static_cast<unsigned long>(rank);
263 mpiio_dump_array<unsigned long>(prefix +
".pref", &offset, 1ul, pref_offset,
265 mpiio_dump_array<int>(prefix +
".id",
id.data(), nlocalpart, offset, MPI_INT);
267 mpiio_dump_array<double>(prefix +
".pos", pos.data(), 3ul * nlocalpart,
268 3ul * offset, MPI_DOUBLE);
270 mpiio_dump_array<double>(prefix +
".vel", vel.data(), 3ul * nlocalpart,
271 3ul * offset, MPI_DOUBLE);
273 mpiio_dump_array<int>(prefix +
".type", type.data(), nlocalpart, offset,
277 std::vector<char> bonds;
281 namespace io = boost::iostreams;
282 io::stream_buffer<io::back_insert_device<std::vector<char>>> os{
283 io::back_inserter(bonds)};
284 boost::archive::binary_oarchive bond_archiver{os};
286 for (
auto const &p : particles) {
287 bond_archiver << p.bonds();
292 auto const bonds_size =
static_cast<unsigned long>(bonds.size());
295 mpiio_dump_array<unsigned long>(prefix +
".boff", &bonds_size, 1ul,
296 pref_offset, MPI_UNSIGNED_LONG);
297 mpiio_dump_array<char>(prefix +
".bond", bonds.data(), bonds.size(),
298 bonds_offset, MPI_CHAR);
310static unsigned long get_num_elem(
const std::string &fn, std::size_t elem_sz) {
315 if (stat(fn.c_str(), &st) != 0) {
316 auto const reason = strerror(errno);
317 fatal_error(
"Could not get file size of", fn, reason);
319 return static_cast<unsigned long>(st.st_size) / elem_sz;
335 std::size_t pref, MPI_Datatype MPI_T) {
338 ret = MPI_File_open(MPI_COMM_WORLD,
const_cast<char *
>(fn.c_str()),
339 MPI_MODE_RDONLY, MPI_INFO_NULL, &f);
344 static_cast<MPI_Offset
>(pref) *
static_cast<MPI_Offset
>(
sizeof(T));
345 ret = MPI_File_set_view(f, offset, MPI_T, MPI_T,
const_cast<char *
>(
"native"),
348 ret |= MPI_File_read_all(f, arr,
static_cast<int>(len), MPI_T,
350 static_cast<void>(ret and
fatal_error(
"Could not read file", fn, &f, ret));
361static unsigned read_head(
const std::string &fn,
int rank) {
362 unsigned n_fields = 0u;
365 f = fopen(fn.c_str(),
"rb");
366 static_cast<void>(not f and
fatal_error(
"Could not open file", fn));
367 auto const n = fread(
static_cast<void *
>(&n_fields),
sizeof n_fields, 1, f);
368 static_cast<void>((n == 1) or
fatal_error(
"Could not read file", fn));
370 MPI_Bcast(&n_fields, 1, MPI_UNSIGNED, 0, MPI_COMM_WORLD);
387static std::tuple<unsigned long, unsigned long>
389 unsigned long nglobalpart) {
390 auto const pref_offset =
static_cast<unsigned long>(rank);
391 unsigned long pref = 0ul;
392 unsigned long nlocalpart = 0ul;
393 mpiio_read_array<unsigned long>(fn, &pref, 1ul, pref_offset,
396 MPI_Send(&pref, 1, MPI_UNSIGNED_LONG, rank - 1, 0, MPI_COMM_WORLD);
398 MPI_Recv(&nlocalpart, 1, MPI_UNSIGNED_LONG, rank + 1, MPI_ANY_TAG,
399 MPI_COMM_WORLD, MPI_STATUS_IGNORE);
401 nlocalpart = nglobalpart;
403 return {pref, nlocalpart};
411 MPI_Comm_size(MPI_COMM_WORLD, &size);
412 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
413 auto const nproc =
get_num_elem(prefix +
".pref",
sizeof(
unsigned long));
414 auto const nglobalpart =
get_num_elem(prefix +
".id",
sizeof(
int));
416 if (rank == 0 && nproc !=
static_cast<unsigned long>(size)) {
417 fatal_error(
"Trying to read a file with a different COMM "
418 "size than at point of writing.");
424 auto const avail_fields =
read_head(prefix +
".head", rank);
425 if (rank == 0 && (fields & avail_fields) != fields) {
426 fatal_error(
"Requesting to read fields which were not dumped.");
433 auto const [pref, nlocalpart] =
434 read_prefs(prefix +
".pref", rank, size, nglobalpart);
436 std::vector<Particle> particles(nlocalpart);
441 std::vector<int> id(nlocalpart);
442 auto id_it =
id.begin();
443 mpiio_read_array<int>(prefix +
".id",
id.data(), nlocalpart, pref, MPI_INT);
445 for (
auto &p : particles) {
454 std::vector<double> pos(3ul * nlocalpart);
455 auto pos_it = pos.begin();
456 mpiio_read_array<double>(prefix +
".pos", pos.data(), 3ul * nlocalpart,
457 3ul * pref, MPI_DOUBLE);
459 for (
auto &p : particles) {
460 std::copy_n(pos_it, 3u, std::begin(p.pos()));
468 std::vector<int> type(nlocalpart);
469 auto type_it = type.begin();
470 mpiio_read_array<int>(prefix +
".type", type.data(), nlocalpart, pref,
473 for (
auto &p : particles) {
482 std::vector<double> vel(3ul * nlocalpart);
483 auto vel_it = vel.begin();
484 mpiio_read_array<double>(prefix +
".vel", vel.data(), 3ul * nlocalpart,
485 3ul * pref, MPI_DOUBLE);
487 for (
auto &p : particles) {
488 std::copy_n(vel_it, 3u, std::begin(p.v()));
496 auto const pref_offset =
static_cast<unsigned long>(rank);
497 unsigned long bonds_size = 0u;
498 mpiio_read_array<unsigned long>(prefix +
".boff", &bonds_size, 1ul,
499 pref_offset, MPI_UNSIGNED_LONG);
504 std::vector<char> bond(bonds_size);
505 mpiio_read_array<char>(prefix +
".bond", bond.data(), bonds_size,
506 bonds_offset, MPI_CHAR);
508 boost::iostreams::array_source src(bond.data(), bond.size());
509 boost::iostreams::stream<boost::iostreams::array_source> ss(src);
510 boost::archive::binary_iarchive ia(ss);
512 for (
auto &p : particles) {
517 for (
auto &p : particles) {
Vector implementation and trait types for boost qvm interoperability.
Data structures for bonded interactions.
int number_of_partners(Bonded_IA_Parameters const &iaparams)
Get the number of bonded partners for the specified bond.
container for bonded interactions.
mapped_type at(key_type const &key) const
bool contains(key_type const &key) const
auto get_next_key() const
base_type::size_type size() const
void errexit()
exit ungracefully, core dump if switched on.
This file contains the errorhandling code for severe errors, like a broken bond or illegal parameter ...
static unsigned long mpi_calculate_file_offset(unsigned long n_items)
Calculate the file offset on the local node.
void mpi_mpiio_common_write(std::string const &prefix, unsigned fields, BondedInteractionsMap const &bonded_ias, ParticleRange const &particles, write_buffers &buffers)
Parallel binary output using MPI-IO.
static std::tuple< unsigned long, unsigned long > read_prefs(const std::string &fn, int rank, int size, unsigned long nglobalpart)
Read the pref file.
static unsigned long get_num_elem(const std::string &fn, std::size_t elem_sz)
Get the number of elements in a file by its file size and elem_sz.
static void mpiio_read_array(const std::string &fn, T *arr, std::size_t len, std::size_t pref, MPI_Datatype MPI_T)
Read a previously dumped array of size len starting from prefix pref of type T using MPI_T as MPI dat...
static void mpiio_dump_array(const std::string &fn, T const *arr, std::size_t len, std::size_t pref, MPI_Datatype MPI_T)
Dump data arr of size len starting from prefix pref of type T using MPI_T as MPI datatype.
static unsigned read_head(const std::string &fn, int rank)
Read the header file and return the first value.
void mpi_mpiio_common_read(const std::string &prefix, unsigned fields, CellStructure &cell_structure)
Parallel binary input using MPI-IO.
static void dump_info(std::string const &fn, unsigned fields, BondedInteractionsMap const &bonded_ias)
Dump the fields and bond information.
static bool fatal_error(char const *msg, std::string const &fn="", std::string const &extra="")
Fatal error handler.
Describes a cell structure / cell system.
Particle * add_particle(Particle &&p)
Add a particle.
void remove_all_particles()
Remove all particles from the cell system.
std::vector< double > pos
std::vector< double > vel