Skip to content

Instantly share code, notes, and snippets.

@psychocoderHPC
Last active January 27, 2017 12:22
Show Gist options
  • Select an option

  • Save psychocoderHPC/b1d0e2d2fc4891e372ecca9cb3a2375d to your computer and use it in GitHub Desktop.

Select an option

Save psychocoderHPC/b1d0e2d2fc4891e372ecca9cb3a2375d to your computer and use it in GitHub Desktop.

This little code snipped messure the performance for adios write/read and verify the results.

compile no transform: mpic++ -std=c++11 main.cpp -I$ADIOS_ROOT/include -L$MPI_ROOT/lib -L$ADIOS_ROOT/lib   -ladios -ladiosread -L$LZ4_ROOT/lib -llz4 -lz -DADIOS_TRANSFORM=\"none\"

compile with LZ4 transform: mpic++ -std=c++11 main.cpp -I$ADIOS_ROOT/include -L$MPI_ROOT/lib -L$ADIOS_ROOT/lib   -ladios -ladiosread -L$LZ4_ROOT/lib -llz4 -lz -DADIOS_TRANSFORM=\"lz4:threshold=2048,lvl=9\"

#include <iostream>
#include <chrono>  
#include <vector>
#include <cstring>
#include <immintrin.h>
//#include "lz4.h"
#include <stdexcept>
#include <algorithm>
#include <mpi.h>
#include <adios.h>
#include <sstream>
#include <adios_read.h>
#include <sys/stat.h>
#include <ftw.h>

constexpr size_t n_inc = 1024lu * 1024lu * 1024lu * 2;
constexpr size_t n_min = 1024lu * 1024lu * 1024lu * 2; // min data
constexpr size_t n_max = 1024lu * 1024lu * 1024lu * 2;
constexpr int rounds = 1;

constexpr double threshold = 0.20;

using ElemType = char;



#ifndef ADIOS_TRANSFORM
// blosc compressors: zlib, lz4, lz4hc, snappy, zstd, blosclz
//#   define ADIOS_TRANSFORM "blosc:threshold=2048,shuffle=bitshuffle,lvl=1,threads=2,compressor=lz4"
//#   define ADIOS_TRANSFORM "none"
#define ADIOS_TRANSFORM "lz4:threshold=2048,lvl=9"
#endif

#define ADIOS_SUCCESS       err_no_error
#define ADIOS_INVALID_HANDLE -1


#define ADIOS_CMD(_cmd)                                                       \
{                                                                             \
    int _err_code = _cmd;                                                     \
    if (_err_code != ADIOS_SUCCESS)                                           \
    {                                                                         \
        std::string errMsg( adios_errmsg() );                                 \
        if( errMsg.empty() ) errMsg = '\n';                                   \
        std::stringstream s;                                                  \
        s << "ADIOS: error at cmd '" << #_cmd                                 \
          << "' (" << _err_code << ", " << adios_errno << ") in "             \
          << __FILE__ << ":" << __LINE__ << " " << errMsg;                    \
        throw std::runtime_error(s.str());                                    \
    }                                                                         \
}

size_t writtenBytes;

int getFileSize(const char*, const struct stat *st, int) {
    writtenBytes += st->st_size;
    return 0;
}

int main()
{

    MPI_Init(NULL, NULL);
    MPI_Comm comm;
    int rank;
    MPI_Comm_dup(MPI_COMM_WORLD, &comm);
    MPI_Comm_rank(comm, &rank);
    int64_t gh;
    size_t writtenByte = 0;

    ADIOS_CMD(adios_init_noxml(comm));
    ADIOS_CMD(adios_declare_group(&gh, "writer", "/", adios_stat_no));


    ADIOS_CMD(adios_select_method(gh,
                                  "POSIX", "", ""));

    std::srand(0);

    for (size_t nElements = n_min; nElements <= n_max; nElements += n_inc) {
        double timeRead = 0.0;
        double timeWrite = 0.0;
        size_t allBytesWritten = 0;
        for (int r = 0; r < rounds; ++r) {
            std::cout << "start round " << r << "/" << int(rounds) << std::endl;
            std::vector<ElemType> mem_src(nElements, 42);
            std::vector<ElemType> mem_dst(nElements, 23);


            char * src_ptr = (char*) mem_src.data();
            char * dst_ptr = (char*) mem_dst.data();

            for (size_t i = 0; i < mem_src.size(); ++i) {
                double rng = (double) std::rand() / (double) RAND_MAX;
                mem_src[i] = i;
                if (rng < threshold)
                    mem_src[i] = std::rand();
            }


            size_t extent = nElements * sizeof (ElemType);
            size_t offset = 0;
            char* dst = (char*) (dst_ptr + offset);

            char* src = (char*) (src_ptr + offset);
            int64_t fh;


            std::cout << "init round " << r << "/" << rounds << " finsished" << std::endl;
            auto startWrite = std::chrono::high_resolution_clock::now();
            // start writing
            // data size + 10%
            size_t memMB = (extent + extent / 10lu) / 1024lu / 1024lu;

            adios_set_max_buffer_size(memMB);
            ADIOS_CMD(adios_open(&fh, "writer", "bpdata.bp", "w", comm));

            if (fh == ADIOS_INVALID_HANDLE)
                throw std::runtime_error("ADIOS: Failed to open file.");

            char ldims[256], gdims[256], offs[256];
            snprintf(ldims, sizeof (ldims), "%llu", extent);
            snprintf(gdims, sizeof (gdims), "%llu", extent);
            snprintf(offs, sizeof (offs), "%llu", 0);

            std::cout << "adios dim = " << ldims << " offset = " << offs << std::endl;

            const int64_t id = adios_define_var(gh, "x", "", adios_byte, ldims, gdims, offs);
            ADIOS_CMD(adios_set_transform(id, ADIOS_TRANSFORM));
            ADIOS_CMD(adios_write_byid(fh, id, src));
            ADIOS_CMD(adios_close(fh));
            auto endWrite = std::chrono::high_resolution_clock::now();
            timeWrite += (double) std::chrono::duration_cast<std::chrono::microseconds>(endWrite - startWrite).count();

            writtenBytes = 0;
            ftw("bpdata.bp.dir", getFileSize, 1);
            std::cout<<"data uncompressed: "<<extent<<" byte -> written: "<<writtenBytes<<" byte"<<std::endl;
            allBytesWritten += writtenBytes;

            auto startRead = std::chrono::high_resolution_clock::now();
            // start reading
            ADIOS_FILE* fp = adios_read_open_file("bpdata.bp", ADIOS_READ_METHOD_BP, comm);
            ADIOS_SELECTION* fSel = adios_selection_boundingbox(1, &offset, &extent);
            adios_schedule_read(fp, fSel, "x", 0, 1, dst);
            ADIOS_CMD(adios_perform_reads(fp, 1));
            ADIOS_CMD(adios_read_close(fp));



            auto endRead = std::chrono::high_resolution_clock::now();
            timeRead += (double) std::chrono::duration_cast<std::chrono::microseconds>(endRead - startRead).count();
            for (size_t i = 0; i < n_max; ++i)
                if (mem_dst[i] != mem_src[i]) {
                    std::cout << "FAIL: at=" << i << " is=" << (int) mem_dst[i] << " should" << (int) mem_src[i] << std::endl;
                    MPI_Finalize();
                    return 0;
                }
        }
        auto avgRead_ms = timeRead / double(rounds);
        auto avgWrite_ms = timeWrite / double(rounds);
        std::cout<<"--------------write------------------"<<std::endl;
        std::cout << "real write " << (double(allBytesWritten) / (avgWrite_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgWrite_ms / 1.0e6 << "s " << (allBytesWritten) / 1024 / 1024 << "MB" << std::endl;
        std::cout << "eff. write " << (double(nElements * sizeof (ElemType)) / (avgWrite_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgWrite_ms / 1.0e6 << "s " << (nElements*sizeof(ElemType)) / 1024 / 1024 << "MB" << std::endl;
        std::cout<<"--------------read------------------"<<std::endl;
        std::cout << "real read " << (double(allBytesWritten) / (avgRead_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgRead_ms / 1.0e6 << "s " << (allBytesWritten) / 1024 / 1024 << "MB" << std::endl;
        std::cout << "eff. read " << (double(nElements * sizeof (ElemType)) / (avgRead_ms / 1.0e6) / 1024. / 1024.) << "MiB/s " << avgRead_ms / 1.0e6 << "s " << (nElements*sizeof(ElemType)) / 1024 / 1024 << "MB" << std::endl;
    }
    adios_finalize(rank);
    MPI_Finalize();
    return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment