This little code snipped messure the performance for adios write/read and verify the results.
compile no transform: mpic++ -std=c++11 main.cpp -I$ADIOS_ROOT/include -L$MPI_ROOT/lib -L$ADIOS_ROOT/lib -ladios -ladiosread -L$LZ4_ROOT/lib -llz4 -lz -DADIOS_TRANSFORM=\"none\"
compile with LZ4 transform: mpic++ -std=c++11 main.cpp -I$ADIOS_ROOT/include -L$MPI_ROOT/lib -L$ADIOS_ROOT/lib -ladios -ladiosread -L$LZ4_ROOT/lib -llz4 -lz -DADIOS_TRANSFORM=\"lz4:threshold=2048,lvl=9\"
#include <iostream>
#include <chrono>
#include <vector>
#include <cstring>
#include <immintrin.h>
//#include "lz4.h"
#include <stdexcept>
#include <algorithm>
#include <mpi.h>
#include <adios.h>
#include <sstream>
#include <adios_read.h>
#include <sys/stat.h>
#include <ftw.h>
constexpr size_t n_inc = 1024lu * 1024lu * 1024lu * 2;
constexpr size_t n_min = 1024lu * 1024lu * 1024lu * 2; // min data
constexpr size_t n_max = 1024lu * 1024lu * 1024lu * 2;
constexpr int rounds = 1;
constexpr double threshold = 0.20;
using ElemType = char;
#ifndef ADIOS_TRANSFORM
// blosc compressors: zlib, lz4, lz4hc, snappy, zstd, blosclz
//# define ADIOS_TRANSFORM "blosc:threshold=2048,shuffle=bitshuffle,lvl=1,threads=2,compressor=lz4"
//# define ADIOS_TRANSFORM "none"
#define ADIOS_TRANSFORM "lz4:threshold=2048,lvl=9"
#endif
#define ADIOS_SUCCESS err_no_error
#define ADIOS_INVALID_HANDLE -1
#define ADIOS_CMD(_cmd) \
{ \
int _err_code = _cmd; \
if (_err_code != ADIOS_SUCCESS) \
{ \
std::string errMsg( adios_errmsg() ); \
if( errMsg.empty() ) errMsg = '\n'; \
std::stringstream s; \
s << "ADIOS: error at cmd '" << #_cmd \
<< "' (" << _err_code << ", " << adios_errno << ") in " \
<< __FILE__ << ":" << __LINE__ << " " << errMsg; \
throw std::runtime_error(s.str()); \
} \
}
size_t writtenBytes;
int getFileSize(const char*, const struct stat *st, int) {
writtenBytes += st->st_size;
return 0;
}
int main()
{
MPI_Init(NULL, NULL);
MPI_Comm comm;
int rank;
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
MPI_Comm_rank(comm, &rank);
int64_t gh;
size_t writtenByte = 0;
ADIOS_CMD(adios_init_noxml(comm));
ADIOS_CMD(adios_declare_group(&gh, "writer", "/", adios_stat_no));
ADIOS_CMD(adios_select_method(gh,
"POSIX", "", ""));
std::srand(0);
for (size_t nElements = n_min; nElements <= n_max; nElements += n_inc) {
double timeRead = 0.0;
double timeWrite = 0.0;
size_t allBytesWritten = 0;
for (int r = 0; r < rounds; ++r) {
std::cout << "start round " << r << "/" << int(rounds) << std::endl;
std::vector<ElemType> mem_src(nElements, 42);
std::vector<ElemType> mem_dst(nElements, 23);
char * src_ptr = (char*) mem_src.data();
char * dst_ptr = (char*) mem_dst.data();
for (size_t i = 0; i < mem_src.size(); ++i) {
double rng = (double) std::rand() / (double) RAND_MAX;
mem_src[i] = i;
if (rng < threshold)
mem_src[i] = std::rand();
}
size_t extent = nElements * sizeof (ElemType);
size_t offset = 0;
char* dst = (char*) (dst_ptr + offset);
char* src = (char*) (src_ptr + offset);
int64_t fh;
std::cout << "init round " << r << "/" << rounds << " finsished" << std::endl;
auto startWrite = std::chrono::high_resolution_clock::now();
// start writing
// data size + 10%
size_t memMB = (extent + extent / 10lu) / 1024lu / 1024lu;
adios_set_max_buffer_size(memMB);
ADIOS_CMD(adios_open(&fh, "writer", "bpdata.bp", "w", comm));
if (fh == ADIOS_INVALID_HANDLE)
throw std::runtime_error("ADIOS: Failed to open file.");
char ldims[256], gdims[256], offs[256];
snprintf(ldims, sizeof (ldims), "%llu", extent);
snprintf(gdims, sizeof (gdims), "%llu", extent);
snprintf(offs, sizeof (offs), "%llu", 0);
std::cout << "adios dim = " << ldims << " offset = " << offs << std::endl;
const int64_t id = adios_define_var(gh, "x", "", adios_byte, ldims, gdims, offs);
ADIOS_CMD(adios_set_transform(id, ADIOS_TRANSFORM));
ADIOS_CMD(adios_write_byid(fh, id, src));
ADIOS_CMD(adios_close(fh));
auto endWrite = std::chrono::high_resolution_clock::now();
timeWrite += (double) std::chrono::duration_cast<std::chrono::microseconds>(endWrite - startWrite).count();
writtenBytes = 0;
ftw("bpdata.bp.dir", getFileSize, 1);
std::cout<<"data uncompressed: "<<extent<<" byte -> written: "<<writtenBytes<<" byte"<<std::endl;
allBytesWritten += writtenBytes;
auto startRead = std::chrono::high_resolution_clock::now();
// start reading
ADIOS_FILE* fp = adios_read_open_file("bpdata.bp", ADIOS_READ_METHOD_BP, comm);
ADIOS_SELECTION* fSel = adios_selection_boundingbox(1, &offset, &extent);
adios_schedule_read(fp, fSel, "x", 0, 1, dst);
ADIOS_CMD(adios_perform_reads(fp, 1));
ADIOS_CMD(adios_read_close(fp));
auto endRead = std::chrono::high_resolution_clock::now();
timeRead += (double) std::chrono::duration_cast<std::chrono::microseconds>(endRead - startRead).count();
for (size_t i = 0; i < n_max; ++i)
if (mem_dst[i] != mem_src[i]) {
std::cout << "FAIL: at=" << i << " is=" << (int) mem_dst[i] << " should" << (int) mem_src[i] << std::endl;
MPI_Finalize();
return 0;
}
}
auto avgRead_ms = timeRead / double(rounds);
auto avgWrite_ms = timeWrite / double(rounds);
std::cout<<"--------------write------------------"<<std::endl;
std::cout << "real write " << (double(allBytesWritten) / (avgWrite_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgWrite_ms / 1.0e6 << "s " << (allBytesWritten) / 1024 / 1024 << "MB" << std::endl;
std::cout << "eff. write " << (double(nElements * sizeof (ElemType)) / (avgWrite_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgWrite_ms / 1.0e6 << "s " << (nElements*sizeof(ElemType)) / 1024 / 1024 << "MB" << std::endl;
std::cout<<"--------------read------------------"<<std::endl;
std::cout << "real read " << (double(allBytesWritten) / (avgRead_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgRead_ms / 1.0e6 << "s " << (allBytesWritten) / 1024 / 1024 << "MB" << std::endl;
std::cout << "eff. read " << (double(nElements * sizeof (ElemType)) / (avgRead_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgRead_ms / 1.0e6 << "s " << (nElements*sizeof(ElemType)) / 1024 / 1024 << "MB" << std::endl;
}
adios_finalize(rank);
MPI_Finalize();
return 0;
}