[mpich-devel] progress in flushall

[Jeff Hammond]

win_flush_all mentions poking the progress engine for async progress
but it won't actually do this if a given process has not initiated any
RMA operations.  This defeats the purpose of using win_flush_all to
drive progress locally.

Maybe I am misreading the code, but I'd sleep better at night if I was
absolutely certain that win_flush_all hit the progress at least once.

Thanks,

Jeff

(the following excerpts are easily found in src/mpid/ch3/src/ch3u_rma_sync.c)

int MPIDI_Win_flush_all(MPID_Win *win_ptr)
{
    int mpi_errno = MPI_SUCCESS;
    int i;
    MPIDI_STATE_DECL(MPIDI_STATE_MPIDI_WIN_FLUSH_ALL);

    MPIDI_RMA_FUNC_ENTER(MPIDI_STATE_MPIDI_WIN_FLUSH_ALL);

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
                        win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

    /* FIXME: Performance -- we should not process the ops separately.
     * Ideally, we should be able to use the same infrastructure that's used by
     * active target to complete all operations. */

    /* Note: Local RMA calls don't poke the progress engine.  This routine
     * should poke the progress engine when the local target is flushed to help
     * make asynchronous progress.  Currently this is handled by Win_flush().
     */
     for (i = 0; i < MPIR_Comm_size(win_ptr->comm_ptr); i++) {
         if (MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[i].rma_ops_list) == NULL)
             continue;
        if (win_ptr->targets[i].remote_lock_state != MPIDI_CH3_WIN_LOCK_NONE) {
            mpi_errno = win_ptr->RMAFns.Win_flush(i, win_ptr);
            if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
        }
    }


int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
{
    int mpi_errno = MPI_SUCCESS;
    int wait_for_rma_done_pkt = 0;
    MPIDI_RMA_Op_t *rma_op;
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FLUSH);

    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FLUSH);

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
                        win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

    /* Check if win_lock was called */
    MPIU_ERR_CHKANDJUMP(win_ptr->targets[rank].remote_lock_state ==
MPIDI_CH3_WIN_LOCK_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

    /* Local flush: ops are performed immediately on the local process */
    if (rank == win_ptr->comm_ptr->rank) {
        MPIU_Assert(win_ptr->targets[rank].remote_lock_state ==
MPIDI_CH3_WIN_LOCK_GRANTED);
        MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->targets[rank].rma_ops_list));

        /* If flush is used as a part of polling for incoming data, we can
         * deadlock, since local RMA calls never poke the progress engine.  So,
         * make extra progress here to avoid this problem. */
        mpi_errno = MPIDI_CH3_Progress_poke();
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
        goto fn_exit;
    }

    /* NOTE: All flush and req-based operations are currently implemented in
       terms of MPIDI_Win_flush.  When this changes, those operations will also
       need to insert this read/write memory fence for shared memory windows. */

    if (win_ptr->shm_allocated == TRUE) {
        OPA_read_write_barrier();
    }

    rma_op = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[rank].rma_ops_list);

    /* If there is no activity at this target (e.g. lock-all was called, but we
     * haven't communicated with this target), don't do anything. */
    if (win_ptr->targets[rank].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED
        && rma_op == NULL)
    {
        goto fn_exit;
    }

-- 
Jeff Hammond
jeff.science at gmail.com