[mpich-discuss] bug in MPI_File_write_all?

[Rob Latham]

On 05/19/2014 04:49 AM, CANELA-XANDRI Oriol wrote:
> Yes, a attach below a test program. The error can be reproduced running it with 9 MPI threads.
>

Thank you for the test case.

I probably fixed this last week, which is why Rajeev no longer sees this 
on his laptop.  It is likely a bug in the way ROMIO flattens darray 
datatypes.

Open-mpi from last week dies with a "signal 7" with this test case.

==rob

> #include <mpi.h>
>
> #include <iostream>
> #include <sstream>
> #include <string>
> #include <vector>
>
> /**
>   * get the number of local rows and columns
>   */
> int getNumroc(int globalSize, int myProc, int nProcs, int blockSize)
> {
>    int myDist = myProc % nProcs;
>    int nBlocks = globalSize / blockSize;
>    int numroc = nBlocks / nProcs;
>    numroc *= blockSize;
>    int extraBlocks = nBlocks % nProcs;
>    if(myDist < extraBlocks)
>    {
>      numroc += blockSize;
>    }
>    else if(myDist == extraBlocks)
>    {
>      numroc += globalSize % blockSize;
>    }
>    return numroc;
> }
>
> int main(int argc, char **argv)
> {
>    //MPI vars
>    bool mpiRoot;
>    int mpiRank;
>    int mpiNumTasks;
>    char hostName[MPI_MAX_PROCESSOR_NAME];
>    int lenHostName;
>    int myProcRow;
>    int myProcCol;
>
>    // Initiate MPI
>    int tmp;
>    tmp = MPI_Init(&argc, &argv);
>    if (tmp != MPI_SUCCESS) {
>      std::cerr << "Error: MPI can not be started. Terminating." << std::endl;
>      MPI_Abort(MPI_COMM_WORLD, 1);
>    }
>    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
>    MPI_Comm_size(MPI_COMM_WORLD, &mpiNumTasks);
>    MPI_Get_processor_name(hostName, &lenHostName);
>
>    if(mpiNumTasks != 9)
>    {
>      std::cerr << "Error: This is a test program designed for running with 9 threads." << std::endl;
>      MPI_Abort(MPI_COMM_WORLD, 1);
>    }
>
>    //Id of the process in a 2d grid
>    myProcRow = mpiRank / 3;
>    myProcCol = mpiRank % 3;
>
>    double *m;                                ///<A pointer to the distributed matrix
>    double *mRead;                                ///<A pointer to the distributed matrix
>
>    int nGlobRows = 5;                        ///<Number of rows of the global matrix
>    int nGlobCols = 4;                        ///<Number of columns of the global matrix
>    int nRows;                                ///<Number of rows of the local matrix
>    int nCols;                                ///<Number of columns of the local matrix
>    int nBlockRows = 1;                       ///<Number of rows of the distributed matrix blocks
>    int nBlockCols = 3;                       ///<Number of columns of the distributed matrix blocks
>
>    nRows = getNumroc(nGlobRows, myProcRow, 3, nBlockRows);
>    nCols = getNumroc(nGlobCols, myProcCol, 3, nBlockCols);
>
>    m = new double[nRows*nCols];
>    mRead = new double[nRows*nCols];
>    for(int i = 0; i < nRows; i++)
>    {
>      for(int j = 0; j < nCols; j++)
>      {
>        m[i*nCols + j] = 1;
>      }
>    }
>
>    for(int repeat = 0; repeat < 10; repeat++)
>    {
>      int dims[] = {nGlobRows, nGlobCols};
>      int dargs[] = {nBlockRows, nBlockCols};
>      int distribs[] = {MPI_DISTRIBUTE_CYCLIC, MPI_DISTRIBUTE_CYCLIC};
>      int dim[] = {3, 3};
>      char nat[] = "native";
>      int rc;
>      MPI_Datatype dcarray;
>      MPI_File cFile;
>      MPI_Status status;
>
>      MPI_Type_create_darray(mpiNumTasks, mpiRank, 2, dims, distribs, dargs, dim, MPI_ORDER_FORTRAN, MPI_DOUBLE, &dcarray);
>      MPI_Type_commit(&dcarray);
>
>      std::stringstream ss;
>      ss << "test_" << repeat << ".bin";
>      std::string fname = ss.str(); //"test.bin";
>      std::vector<char> fn(fname.begin(), fname.end());
>      fn.push_back('\0');
>      MPI_File_delete (&fn[0], MPI_INFO_NULL);
>
>      //Write file
>      rc = MPI_File_open(MPI_COMM_WORLD, &fn[0], MPI_MODE_EXCL | MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &cFile);
>      if(rc){
>        std::cerr << "Error: Failed to open file." << std::endl;
>        MPI_Abort(MPI_COMM_WORLD, 1);
>      }
>      else
>      {
>        MPI_File_set_view(cFile, 0, MPI_DOUBLE, dcarray, nat, MPI_INFO_NULL);
>        MPI_File_write_all(cFile, m, nRows*nCols, MPI_DOUBLE, &status);
>      }
>      MPI_Barrier(MPI_COMM_WORLD);
>      MPI_File_close(&cFile);
>
>
>      //Initialize matrix before reading to 0;
>      for(int i = 0; i < nRows; i++)
>      {
>        for(int j = 0; j < nCols; j++)
>        {
>          mRead[i*nCols + j] = 0;
>        }
>      }
>      //Read file
>      rc = MPI_File_open(MPI_COMM_WORLD, &fn[0], MPI_MODE_RDONLY, MPI_INFO_NULL, &cFile);
>      if(rc){
>        std::cerr << "Error: Failed to open file." << std::endl;
>        MPI_Abort(MPI_COMM_WORLD, 1);
>      }
>      else
>      {
>        MPI_File_set_view(cFile, 0, MPI_DOUBLE, dcarray, nat, MPI_INFO_NULL);
>        MPI_File_read_all(cFile, mRead, nRows*nCols, MPI_DOUBLE, &status);
>      }
>      MPI_Barrier(MPI_COMM_WORLD);
>      MPI_File_close(&cFile);
>      MPI_Type_free(&dcarray);
>
>      //Check data
>      for(int i = 0; i < nRows; i++)
>      {
>        for(int j = 0; j < nCols; j++)
>        {
>          if(mRead[i*nCols + j] != 1)
>          {
>            std::cerr << "Error in data. " << repeat << " iteration." << std::endl;
>            MPI_Abort(MPI_COMM_WORLD, 1);
>          }
>        }
>      }
>    }
>
>    delete [] m;
>    delete [] mRead;
>
>    return 0;
> }
>
> Oriol
>
>

-- 
Rob Latham
Mathematics and Computer Science Division
Argonne National Lab, IL USA