/* Code for generating deadlocks in MPI
   Aaron Weeden, NCSI, June 29, 2011
   Based on code from Tom Murphy, NCSI
 
   This code is designed to illustrate the different modes MPI provides for 
   sending and receiving messages.  Experiment with different send modes, 
   ordering of the sends and receives, and message sizes to see how each 
   of them behave.
 
*/ 

#include <mpi.h> /* for MPI functions and constants */
#include <stdio.h> /* for printf, sprintf */
#include <stdlib.h> /* for exit */
#include <unistd.h> /* for getopt */

#define MAX_ARRAY_LENGTH 800000

/* For MPI sends and receives, arbitrary. */
#define TAG 42

/* Send modes */
#define BUFFERED_MODE 'b' /* buffered */
#define READY_MODE 'r' /* ready */
#define SYNCRONOUS_MODE 's' /* synchronous */
#define STANDARD_MODE 'v' /* standard */

/* Order of sends and receives */
#define BOTH_SEND 's' /* both send and then both receive */
#define BOTH_RECEIVE 'r' /* both receive and then both send */
#define ALTERNATE_SEND_RECEIVE 'a' /* alternate */

void send(int my_rank, int number_of_processes, int message_length, 
  int send_mode); /* Send an arbitrary message using the given mode */
void recv(int my_rank, int number_of_processes, int message_length);/* Receive
                                                              * a message */
void print_usage(char *message, char *program); /* Print a usage message and
                                                 * exit */

int main(int argc, char** argv) {
    /* Deadlock data */
    int send_mode = BUFFERED_MODE; /* See "Send Modes" above */
    int order = BOTH_SEND; /* See "Order of sends and receives" above */
    int* buffer; /* For buffered sends */
    int message_length = MAX_ARRAY_LENGTH - 4; /* For buffered sends */

    /* Argument parsing information */
    int next_char; /* the next character on the command line */
    char* optstring = "m:o:"; /* the possible values of command line args:
                               *  m -- the send mode (see "Send Modes" above)
                               *  o -- the order (see "Order of Send/Recv"
                               *        above) */

    /* MPI information */
    int my_rank; /* the rank of the MPI process */
    int number_of_processes; /* the size of the MPI world */

    /* Miscellaneous */
    char* errbuf; /* holds error messages */

    /* Initialize MPI */
    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &number_of_processes);

    /* Parse command line arguments (see `man 3 getopt` for details) */
    while ((next_char = getopt(argc, argv, optstring)) != -1) {
        switch(next_char) {
            case 'm':
                send_mode = *optarg;
                break;
            case 'o':
                order = *optarg;
                break;
            default:
                sprintf(errbuf, "Unknown option %d", *optarg);
                print_usage(errbuf, argv[0]);
                break;
        }
        argc -= optind;
        argv += optind;
    }

    /* Set up the buffer if we are doing buffered sends */
    if (send_mode == BUFFERED_MODE) {
        /* The length of the buffer is the amount of overhead plus
         * (the length of the actual message times the size of an int) */
        buffer = (int *)malloc(MPI_BSEND_OVERHEAD + message_length 
          * sizeof(int));

        /* Create the MPI Buffer */
        MPI_Buffer_attach(buffer, MPI_BSEND_OVERHEAD + message_length);
    }

    /* If the order is Both Send, or if the order is Alternating
     * and we are rank 0, send and then receive */
    if ((order == BOTH_SEND) || 
      ((order == ALTERNATE_SEND_RECEIVE) && (my_rank == 0))) {
        send(my_rank, number_of_processes, message_length, send_mode);
        recv(my_rank, number_of_processes, message_length);

    } else { /* Otherwise, receive and then send */
        recv(my_rank, number_of_processes, message_length);
        send(my_rank, number_of_processes, message_length, send_mode);
    }

    /* Clean up MPI and then exit */
    MPI_Finalize();
    return 0;
}

/* Send an arbitrary message using the given mode */
void send(int my_rank, int number_of_processes, int message_length, 
  int send_mode) {
    int return_value; /* Return value of MPI Send */
    int buffer_out[MAX_ARRAY_LENGTH]; /* Message to send */

    printf("Rank %d starting to send\n", my_rank);

    /* Do an MPI Send of the given mode */
    switch(send_mode) {
        case BUFFERED_MODE:
            return_value = MPI_Bsend(buffer_out, message_length, MPI_INT,
              1 - my_rank, TAG, MPI_COMM_WORLD);
            break;
        case READY_MODE:
            return_value = MPI_Rsend(buffer_out, message_length, MPI_INT,
              1 - my_rank, TAG, MPI_COMM_WORLD);
            break;
        case SYNCRONOUS_MODE:
            return_value = MPI_Ssend(buffer_out, message_length, MPI_INT,
        	  1 - my_rank, TAG, MPI_COMM_WORLD);
            break;
        case STANDARD_MODE:
            return_value = MPI_Send(buffer_out, message_length, MPI_INT,
        	  1 - my_rank, TAG, MPI_COMM_WORLD);
            break;
    }

    printf("Rank %d done sending\n", my_rank);
}

/* Receive a message */
void recv(int my_rank, int number_of_processes, int message_length) {
    MPI_Status status; /* Dummy variable needed for MPI_Recv */
    int return_value; /* Return value of MPI_Recv */
    int buffer_in[MAX_ARRAY_LENGTH]; /* The message to be received */

    printf("Rank %d starting to receive\n", my_rank);

    return_value = MPI_Recv(buffer_in, MAX_ARRAY_LENGTH, MPI_INT, 1 - my_rank, 
      TAG, MPI_COMM_WORLD, &status);

    printf("Rank %d done receiving\n", my_rank);
}

/* Usage Message */
void print_usage(char* message, char* program) {
    printf("\n%s\nUsage: %s [-o order] [-l message_length] [-m send_mode]\n",
            message, program);
    printf("\twhere\n");
    printf("\t\torder is R(receive first), S(send first), or A(alternate)\n");
    printf("\t\tmessage_length can be 1 to %d inclusive\n", MAX_ARRAY_LENGTH);
    printf("\t\tmode is B(buffered), R(ready), S(synchronous), or V(standard)\n");
    exit(1);
}