
//#define GNU_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <endian.h>
#include <infiniband/verbs.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include "cecilie.h"

/* UDP Ports */
#define CECILIE_ROCE_PORT	4791
#define CECILIE_CONTROL_PORT    0xDFCC
#define LOCAL_PORT		0xDFCD

/* RDMA Defines */
#define CECILIE_QKEY		0xffffffff
#define RDMA_RX_WR_DEPTH	30000
#define RDMA_RECV_WRID		1
#define DEFAULT_PKT_SIZE	4096

#define NAME_MAX_LEN		100
#define DISCOVER_TIMEOUT_MS	500

#define DISCARD_STDIN	{for(char x; (x = getchar()) != EOF && x != '\n';){}}
#define USE_EVENTS


struct rdma_context_t{
        char dev_name[NAME_MAX_LEN];
	char file_name[NAME_MAX_LEN];
        uint64_t pkt_size;
        uint64_t pkt_cnt;
        struct ibv_context      *context;
        struct ibv_comp_channel *channel;
        struct ibv_pd           *pd;
        struct ibv_mr           *mr;
        struct ibv_cq           *cq;
        struct ibv_qp           *qp;
        struct ibv_sge          *sge_list;
        uint8_t                 *buf;
        int                     rx_depth;
        int                     pending;
	uint32_t 		qp_num;
	uint32_t 		q_key;
	uint32_t 		psn;
        struct ibv_port_attr    portinfo;

};

struct cecilie_app_context_t{
	cecilie_t *dev_list;
	cecilie_t *dev;
	cecilie_reg_t *regs;
};

void print_regs(cecilie_reg_t *regs, int reg_cnt){
	for(int i = 0; i < reg_cnt; i++){
		printf("reg: %08x	val: %08x\n", regs[i].reg_addr, regs[i].reg_val);
	}
}

#define TIME_MS(time) (time.tv_sec*1000 + time.tv_usec/1000)

int find_cecilie(char *if_name, cecilie_t **list)
{
        int skt = 1;
        int opt = 1;
        uint32_t msg[1] = {MSG_INFO_HEADER};
        uint32_t resp[5];
        struct sockaddr_in cecilie_net;
        struct sockaddr_in adapter;
	struct timeval tv;
	struct timeval start;
	struct timeval end;
	int cnt = 0;
	cecilie_t *cecilie = NULL;
	tv.tv_sec = 0;
        tv.tv_usec = 500*1000;

	/* Create and setup socket */
        skt = socket(AF_INET, SOCK_DGRAM, 0);
        if(skt < 0){
                printf("Failed to create socket\n");
                return 0;
        }
        setsockopt(skt, SOL_SOCKET, (SO_BROADCAST), &opt, sizeof(opt));
        setsockopt(skt, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));
	
	/* Bind socket with adapter */
	adapter.sin_port = ntohs(LOCAL_PORT);
	adapter.sin_family = AF_INET;
	get_mac_ip(if_name, NULL, &adapter.sin_addr);
	if(bind(skt, (struct sockaddr *)&adapter, sizeof(adapter))){
		close(skt);
		printf("Failed to bind socket to interface %s\n", if_name);
		return 0;
	}

	/* Send discovery broadcast */
        cecilie_net.sin_addr.s_addr = htonl(INADDR_BROADCAST);
        cecilie_net.sin_port = ntohs(CECILIE_CONTROL_PORT);
        cecilie_net.sin_family = AF_INET;
        opt = sendto(skt, (char*)msg, sizeof(msg), 0, (struct sockaddr*)&cecilie_net, sizeof(cecilie_net));

	if(opt == -1){
                close(skt);
                printf("Failed to send packet: %s\n", strerror(errno));
                return 0;
        }

        if(opt != sizeof(msg)){
                close(skt);
                printf("Wrong message length\n");
                return 0;
        }
		
	/* Receive responses until timeout runs out */
	gettimeofday(&start, NULL);
	gettimeofday(&end, NULL);
	timersub(&end, &start, &tv);
	while(TIME_MS(tv) < DISCOVER_TIMEOUT_MS) {
		opt = recv(skt, (char*)resp, sizeof(resp), 0);
		gettimeofday(&end, NULL);
		timersub(&end, &start, &tv);
	
		if(opt == -1)
	        {
#if 0
			if(cnt == 0)
	               		printf("Error on recv: %s\n", strerror(errno));
#endif
			continue;
		}
	
	        if(opt != sizeof(resp))
	        {
	                printf("Message with wrong length received [%d]\n", opt);
	        	continue;
		}
	
	        if(resp[0] == MSG_RESP_HEADER)
	        {
			cecilie = malloc(sizeof(cecilie_t));
			if(!cecilie)
				continue;
	                memcpy(&cecilie->net, (char*)&resp[1], sizeof(struct cecilie_net_info_t));
	                /* Change byteorder for IP addresses */
	                cecilie->net.ip_addr.s_addr = ntohl(cecilie->net.ip_addr.s_addr);
	                cecilie->net.net_mask.s_addr = ntohl(cecilie->net.net_mask.s_addr);
			cecilie->next = NULL;
	                /* swap bytes for mac address */
	                for (int i = 0; i < 3; i++) {
	                        unsigned char tmp;
	                        tmp = cecilie->net.mac_addr[i];
	                        cecilie->net.mac_addr[i] = cecilie->net.mac_addr[5 - i];
	                        cecilie->net.mac_addr[5 - i] = tmp;
	                }
			/* Save interface name */
			sprintf(cecilie->adapter_name, "%.*s", sizeof(cecilie->adapter_name)-1, if_name);
			insert_cecilie_to_list(cecilie, list);
	        }else{
	                printf("Wrong message header received\n");
			continue;
		}

#if 0
	        printf("MAC:       %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx\n",
	                cecilie->net.mac_addr[5], cecilie->net.mac_addr[4], cecilie->net.mac_addr[3],
	                cecilie->net.mac_addr[2], cecilie->net.mac_addr[1], cecilie->net.mac_addr[0]);
	        printf("IP:        %s\n", inet_ntoa(cecilie->net.ip_addr));
	        printf("Net mask:  %s\n\n", inet_ntoa(cecilie->net.net_mask));
#endif

		/* upgrade board counter and get actual time */
		cnt++;
	}

	printf("Found %d boards on inteface %s\n", cnt, if_name);

        close(skt);
        return cnt;
}

void print_interfaces(char **if_names, int if_num)
{
        struct sockaddr_in adapter;
	unsigned char mac[6];

	if(if_names == NULL)
	{
		printf("There is no adapter available\n");
	}

	for(int i = 0; i < if_num; i++)
	{
		if(if_names[i] != NULL){
			get_mac_ip(if_names[i], mac, &adapter.sin_addr);
			printf("%-20s", if_names[i]);
			printf("%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx\t",
	                	mac[5], mac[4], mac[3],
	                	mac[2], mac[1], mac[0]);
	        	printf("%s\n", inet_ntoa(adapter.sin_addr));
		}
	}
}

int write_cecilie_reg(cecilie_t *cecilie, uint32_t reg_addr, uint32_t reg_val)
{
        int skt = 1;
        int opt = 1;
        uint32_t msg[3];
        uint32_t resp[3];
        struct sockaddr_in cecilie_net;
        struct timeval tv;
        tv.tv_sec = 1;
        tv.tv_usec = 0;

        /* Create and setup socket */
        skt = socket(AF_INET, SOCK_DGRAM, 0);
        if(skt < 0){
                printf("Socket error\n");
                return 1;
        }
        setsockopt(skt, SOL_SOCKET, (SO_BROADCAST), &opt, sizeof(opt));
        opt = 1000;
        setsockopt(skt, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));

        cecilie_net.sin_addr.s_addr = cecilie->net.ip_addr.s_addr;
        cecilie_net.sin_port = ntohs(CECILIE_CONTROL_PORT);
        cecilie_net.sin_family = AF_INET;

        /* Write to registe */
	msg[0] = MSG_WRITE_HEADER;
	msg[1] = reg_addr;
	msg[2] = reg_val;

       	/* Send message */
       	opt = sendto(skt, (char*)msg, sizeof(msg), 0, (struct sockaddr*)&cecilie_net, sizeof(cecilie_net));

        /* Check if error occured */
        if(opt == -1){
                close(skt);
                printf("Error on sendto: %s\n", strerror(errno));
                return 1;
        }

        /* Check if sent length is equal to desired length */
        if(opt != sizeof(msg)){
                close(skt);
                printf("Nothing send\n");
                return 1;
        }

        /* Receive response */
        opt = recv(skt, (char*)resp, sizeof(resp), 0);

        /* Check if error occured */
        if(opt == -1){
                printf("Error on recv: %s\n", strerror(errno));
                close(skt);
                return 1;
        }

        /* Check if received message has desired length */
        if(opt != sizeof(resp)){
                close(skt);
                printf("Message with wrong length received\n");
                return 1;
        }

        /* Check message header */
        if(resp[0] != MSG_RESP_HEADER){
                printf("Wrong message header received\n");
                close(skt);
                return 1;
        }

        /* Check message */
        if(resp[1] == reg_addr){
                printf("Register %8x: write OK\n", reg_addr);
        }else if(resp[1] == ERR_REG_NOT_EXIST){
                printf("Register %8x: register does not exist\n", reg_addr);
        }else{
                printf("Register %8x: invalid respond [%8x] !!!!\n", reg_addr, resp[1]);
        }

        close(skt);
        return 0;
}


int write_cecilie_config(cecilie_t *cecilie, cecilie_reg_t *config, int reg_cnt)
{
        int skt = 1;
        int opt = 1;
        uint32_t msg[15];
        uint32_t resp[15];
        struct sockaddr_in cecilie_net;
        struct timeval tv;
        tv.tv_sec = 1;
        tv.tv_usec = 0;
        int send_cnt = 0;

        /* Create and setup socket */
        skt = socket(AF_INET, SOCK_DGRAM, 0);
        if(skt < 0){
                printf("Socket error\n");
                return 1;
        }
        setsockopt(skt, SOL_SOCKET, (SO_BROADCAST), &opt, sizeof(opt));
        opt = 1000;
        setsockopt(skt, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));

        cecilie_net.sin_addr.s_addr = cecilie->net.ip_addr.s_addr;
        cecilie_net.sin_port = ntohs(CECILIE_CONTROL_PORT);
        cecilie_net.sin_family = AF_INET;

	printf("Writing configuration to cecilie\n");
        /* Write configuration - max 7 registers each send operation */
        for(int i = 0; i < reg_cnt; i += 7)
        {
                /* Get message length and fill send buffer */
                send_cnt = (i + 7 < reg_cnt) || (reg_cnt % 7 == 0) ? 7 : reg_cnt % 7;
                msg[0] = MSG_WRITE_HEADER;
                memcpy((char*)&msg[1], (char*)&config[i], send_cnt*sizeof(*config));

                /* Send message */
                opt = sendto(skt, (char*)msg, sizeof(uint32_t) + send_cnt*sizeof(*config), 0, (struct sockaddr*)&cecilie_net, sizeof(cecilie_net));

                /* Check if error occured */
                if(opt == -1){
                        close(skt);
                        printf("Error on sendto: %s\n", strerror(errno));
                        return 1;
                }

                /* Check if sent length is equal to desired length */
                if(opt != sizeof(uint32_t) + send_cnt*sizeof(*config)){
                        close(skt);
                        printf("Nothing send\n");
                        return 1;
                }

                /* Receive response */
                opt = recv(skt, (char*)resp, sizeof(resp), 0);

                /* Check if error occured */
                if(opt == -1){
                        printf("Error on recv: %s\n", strerror(errno));
                        close(skt);
                        return 1;
                }

                /* Check if received message has desired length */
                if(opt != send_cnt*sizeof(*config) + sizeof(uint32_t)){
                        close(skt);
                        printf("Message with wrong length received %d\n", opt);
                        return 1;
                }

                /* Check message header */
                if(resp[0] != MSG_RESP_HEADER){
                        printf("Wrong message header received\n");
                        close(skt);
                        return 1;
                }

                /* Check message */
#if 0    
    		for(int j = 0; j < send_cnt; j++){
                        if(resp[2*j+1] == config[i+j].reg_addr){
                                printf("Register %8x: write OK\n", config[i+j].reg_addr);
                        }else if(resp[2*j+1] == ERR_REG_NOT_EXIST){
                                printf("Register %8x: register does not exist\n", config[i+j].reg_addr);
                        }else{
                                printf("Register %8x: invalid respond [%8x] !!!!\n", config[i+j].reg_addr, resp[2*j+1]);
                        }
                }
#endif
	}

        close(skt);
        return 0;
}


int read_cecilie_reg(cecilie_t *cecilie, uint32_t reg_addr, uint32_t *reg_val)
{
        int skt = 1;
        int opt = 1;
        uint32_t msg[2];
        uint32_t resp[3];
        struct sockaddr_in cecilie_net;
        struct timeval tv;
        tv.tv_sec = 1;
        tv.tv_usec = 0;


        /* Create and setup socket */
        skt = socket(AF_INET, SOCK_DGRAM, 0);
        if(skt < 0){
                printf("Socket error\n");
                return 1;
        }
        setsockopt(skt, SOL_SOCKET, (SO_BROADCAST), &opt, sizeof(opt));
        opt = 1000;
        setsockopt(skt, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));

        cecilie_net.sin_addr.s_addr = cecilie->net.ip_addr.s_addr;
        cecilie_net.sin_port = ntohs(CECILIE_CONTROL_PORT);
        cecilie_net.sin_family = AF_INET;

        /* Write to registe */
	msg[0] = MSG_READ_HEADER;
	msg[1] = reg_addr;

       	/* Send message */
       	opt = sendto(skt, (char*)msg, sizeof(msg), 0, (struct sockaddr*)&cecilie_net, sizeof(cecilie_net));

        /* Check if error occured */
        if(opt == -1){
                close(skt);
                printf("Error on sendto: %s\n", strerror(errno));
                return 1;
        }

        /* Check if sent length is equal to desired length */
        if(opt != sizeof(msg)){
                close(skt);
                printf("Nothing send\n");
                return 1;
        }

        /* Receive response */
        opt = recv(skt, (char*)resp, sizeof(resp), 0);

        /* Check if error occured */
        if(opt == -1){
                printf("Error on recv: %s\n", strerror(errno));
                close(skt);
                return 1;
        }

        /* Check if received message has desired length */
        if(opt != sizeof(resp)){
                close(skt);
                printf("Message with wrong length received\n");
                return 1;
        }

        /* Check message header */
        if(resp[0] != MSG_RESP_HEADER){
                printf("Wrong message header received\n");
                close(skt);
                return 1;
        }

        /* Check message */
        if(resp[1] == reg_addr){
                printf("Register %8x: %08x\n", reg_addr, resp[2]);
        }else if(resp[1] == ERR_REG_NOT_EXIST){
                printf("Register %8x: register does not exist\n", reg_addr);
        }else{
                printf("Register %8x: invalid respond [%8x] !!!!\n", reg_addr, resp[1]);
        }

	*reg_val = resp[2];

        close(skt);
        return 0;
}


int read_cecilie_config(cecilie_t *cecilie, cecilie_reg_t *config, int reg_cnt)
{
        int skt = 1;
        int opt = 1;
        uint32_t msg[15];
        uint32_t resp[15];
        struct sockaddr_in cecilie_net;
        struct timeval tv;
        tv.tv_sec = 1;
        tv.tv_usec = 0;
        int send_cnt = 0;

        /* Create and setup socket */
        skt = socket(AF_INET, SOCK_DGRAM, 0);
        if(skt < 0){
                printf("Socket error\n");
                return 1;
        }
        setsockopt(skt, SOL_SOCKET, (SO_BROADCAST), &opt, sizeof(opt));
        opt = 1000;
        setsockopt(skt, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));

        cecilie_net.sin_addr.s_addr = cecilie->net.ip_addr.s_addr;
        cecilie_net.sin_port = ntohs(CECILIE_CONTROL_PORT);
        cecilie_net.sin_family = AF_INET;

        /* Write configuration - max 7 registers each send operation */
        for(int i = 0; i < reg_cnt; i += 7)
        {
                /* Get message length and fill send buffer */
                send_cnt = (i + 7 < reg_cnt) || (reg_cnt % 7 == 0) ? 7 : reg_cnt % 7;
                msg[0] = MSG_READ_HEADER;
		for(int j = 0; j < send_cnt; j++){
                	msg[j+1] = config[i+j].reg_addr;
		}
                /* Send message */
                opt = sendto(skt, (char*)msg, sizeof(uint32_t) + send_cnt*sizeof(config[0].reg_addr), 0, (struct sockaddr*)&cecilie_net, sizeof(cecilie_net));

                /* Check if error occured */
                if(opt == -1){
                        close(skt);
                        printf("Error on sendto: %s\n", strerror(errno));
                        return 1;
                }

                /* Check if sent length is equal to desired length */
                if(opt != sizeof(uint32_t) + send_cnt*sizeof(config[0].reg_addr)){
                        close(skt);
                        printf("Nothing send\n");
                        return 1;
                }

                /* Receive response */
                opt = recv(skt, (char*)resp, sizeof(resp), 0);

                /* Check if error occured */
                if(opt == -1){
                        printf("Error on recv: %s\n", strerror(errno));
                        close(skt);
                        return 1;
                }

                /* Check if received message has desired length */
                if(opt != send_cnt*sizeof(*config) + sizeof(uint32_t)){
                        close(skt);
                        printf("Message with wrong length received\n");
                        return 1;
                }

                /* Check message header */
                if(resp[0] != MSG_RESP_HEADER){
                        printf("Wrong message header received\n");
                        close(skt);
                        return 1;
                }

                /* Check message */
                for(int j = 0; j < send_cnt; j++){
                        if(resp[1+2*j] == ERR_REG_UNKNOWN){
                                printf("Register %08x: register does not exist %08x\n", config[i+j].reg_addr, resp[1+2*j]);
                        }else{
                                printf("Register %08x: %08x\n", config[i+j].reg_addr, resp[2*j+2]);
				config[i+j].reg_val = resp[2*j+2];
                        }
                }
        }

        close(skt);
        return 0;
}


char get_opt()
{
	char opt;

	opt = getchar();
	while(opt == '\n'){
		opt = getchar();
	}

	while(getchar() != '\n'){}

	return opt;
}


void help()
{
	printf(	"Cecilie control help:\n"
		"'h'	Print this help\n"
		"'d'	Register description\n"
		"'f'	Find cecilie\n"
		"'l'	Print cecilie list\n"
		"'c'	Choose board\n"
		"'w'	Write register(s)\n"
		"'r'	Read register(s)\n"
		"'t'	Start RDMA transfer\n"
		"'e'	Exit\n");
}

int file_read_config(char *file_name, cecilie_reg_t **config){

	FILE *fd = NULL;
	uint32_t reg_addr, reg_val = 0;
	int cnt = 0, tmp = 0;
	cecilie_reg_t *cfg = NULL;

	if(config == NULL)
		return 0;
	
	if(*config != NULL)
		free(*config);

	if(!file_name)
		return 0;

	fd = fopen(file_name, "r");
	if(!fd)
		return 0;

	while(fscanf(fd, "%x, %x\n", &reg_addr, &reg_val) == 2){
		cnt++;
	}

	cfg = malloc(cnt*sizeof(cecilie_reg_t));
	if(!cfg)
		return 0;

	rewind(fd);
	while(fscanf(fd, "%x, %x", &reg_addr, &reg_val) == 2 && tmp < cnt){
		cfg[tmp].reg_addr = reg_addr;
		cfg[tmp].reg_val = reg_val;
		tmp++;
	}

	fclose(fd);
	*config = cfg;

	return tmp;
}

int file_read_regs(char *file_name, cecilie_reg_t **config){

	FILE *fd = NULL;
	uint32_t reg_addr = 0;
	int cnt = 0, tmp = 0;
	cecilie_reg_t *cfg = NULL;

	if(config == NULL)
		return 0;
	
	if(*config != NULL)
		free(*config);

	if(!file_name)
		return 0;

	fd = fopen(file_name, "r");
	if(!fd)
		return 0;

	while(fscanf(fd, "%x%*[^\n]", &reg_addr) == 1){
		cnt++;
	}

	cfg = malloc(cnt*sizeof(cecilie_reg_t));
	if(!cfg)
		return 0;

	rewind(fd);
	while(fscanf(fd, "%x%*[^\n]", &reg_addr) == 1 && tmp < cnt){
		cfg[tmp].reg_addr = reg_addr;
		cfg[tmp].reg_val = 0;
		tmp++;
	}

	fclose(fd);
	*config = cfg;

	return tmp;
}

int file_write_regs(char *file_name, cecilie_reg_t *config, int config_size){

	FILE *fd = NULL;

	if(config == NULL)
		return 1;
	
	if(!file_name)
		return 1;

	if(config_size == 0)
		return 1;

	fd = fopen(file_name, "w");
	if(!fd)
		return 1;

	for(int i = 0; i < config_size; i++){
		fprintf(fd, "%08x, %08x\n", config[i].reg_addr, config[i].reg_val);
	}

	fclose(fd);

	return 0;
}



void print_config(cecilie_reg_t *config, int cnt)
{	
	if(!config)
		return;
	printf("Register\tValue\n");
	for(int i = 0; i < cnt; i++){
		printf("%08x\t%08x\n", config[i].reg_addr, config[i].reg_val);	
	}

}

void set_reg(cecilie_reg_t *regs, int reg_cnt, uint32_t reg_addr, uint32_t reg_val){
	

	for(int i = 0; i < reg_cnt; i++){
		if(regs[i].reg_addr == reg_addr){
			regs[i].reg_val = reg_val;
			break;
		}
	}
}



int find_rdma_device(char *eth_if_name, char *rdma_name, int name_max_size)
{
        char path[] = "/sys/class/infiniband/";
        char dev_path[300];
        char parent[300];
        DIR *dr = NULL;
        DIR *dev_dr = NULL;
        struct dirent *de;
        struct dirent *dev_de;
        char *ptr = NULL;
        /* Open infiniband directory */
        dr = opendir(path);
        if(dr == NULL){
                printf("Couldn't open directory %s\n", path);
                return 1;
        }

        /* Go through all infiniband devices and check if its net name is
         * equal to ethernet interface name */
        while((de = readdir(dr)) != NULL){
		/* Check if folder name is not current folder or path backwards */
                if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
                        continue;
                /* Open directory device/net of the found device */
                snprintf(dev_path, sizeof(dev_path), "%s%s/device/net", path, de->d_name);
                dev_dr = opendir(dev_path);
                if(!dev_dr){
                        printf("Couldn't open directory %s\n", dev_path);
                        continue;
                }
		/* Go through all folder names in dev_dr */
		while((dev_de = readdir(dev_dr)) != NULL){
			/* Check if folder name is not current folder or path backwards */
                	if(strcmp(dev_de->d_name, ".") == 0 || strcmp(dev_de->d_name, "..") == 0)
                        	continue;
			
             		/* Compare net device name with ethernet interface name */
             	  	if(strcmp(eth_if_name, dev_de->d_name) == 0){
             	  	        strncpy(rdma_name, de->d_name, name_max_size);
				closedir(dev_dr);
             	  	        closedir(dr);
             	  	        return 0;
             	  	}
		}
                closedir(dev_dr);
        }
        closedir(dr);
        return 1;
}



static int post_rdma_recv(struct rdma_context_t *ctx, int posted, int total_cnt, int post_cnt)
{
	struct ibv_recv_wr wr;
	struct ibv_recv_wr *bad_wr;

	if(ctx == NULL){
		return -1;
	}

	if(ctx->sge_list == NULL){
		return -1;
	}

	if(posted >= total_cnt){
		return 0;
	}

	if(ctx->qp == NULL){
		return -1;
	}

	/* How many work requests should be posted? */
	post_cnt = (total_cnt - posted) < post_cnt ? total_cnt - posted : post_cnt;

	wr.wr_id = RDMA_RECV_WRID;
	wr.next = NULL;
	wr.num_sge = 1;
	for(int i = 0; i < post_cnt; i++){
		wr.sg_list = &ctx->sge_list[posted++];

		if (ibv_post_recv(ctx->qp, &wr, &bad_wr)){
			printf("ibv_post_recv error: %s\n", strerror(errno));
			return i;
		}
	}

	return post_cnt;
}


int close_rdma_ctx(struct rdma_context_t *ctx)
{
	if (ibv_destroy_qp(ctx->qp)) {
		fprintf(stderr, "Couldn't destroy QP\n");
		return 1;
	}

	if (ibv_destroy_cq(ctx->cq)) {
		fprintf(stderr, "Couldn't destroy CQ\n");
		return 1;
	}

	if (ibv_dereg_mr(ctx->mr)) {
		fprintf(stderr, "Couldn't deregister MR\n");
		return 1;
	}

	if (ibv_dealloc_pd(ctx->pd)) {
		fprintf(stderr, "Couldn't deallocate PD\n");
		return 1;
	}
#ifdef USE_EVENTS
	if (ibv_destroy_comp_channel(ctx->channel)) {
		fprintf(stderr, "Couldn't destroy completion channel\n");
		return 1;
	}
#endif
	if (ibv_close_device(ctx->context)) {
		fprintf(stderr, "Couldn't release context\n");
		return 1;
	}

	free(ctx->sge_list);
	free(ctx->buf);

	return 0;
}


struct ibv_sge* prepare_sge_list(uint32_t sge_size, int sge_cnt, uint32_t lkey, uint8_t *buf)
{
        struct ibv_sge *list = NULL;

        list = malloc(sizeof(struct ibv_sge)*sge_cnt);
        if(!list){
                printf("Couldn't allocate buffer for ibv_sge list\n");
                return NULL;
        }

        for(uint64_t i = 0; i < sge_cnt; i++){
                list[i].addr = (uintptr_t)&buf[(uint64_t)i*sge_size];
                list[i].length = sge_size;
                list[i].lkey = lkey;
        }

        return list;
}


/* pkt_size, pkt_cnt and rx_depth of rdma_context should be already set */
int init_rdma_ctx(struct rdma_context_t *ctx)
{
	uint64_t sge_size;
	uint64_t buf_size;
	int port = 1;
	struct ibv_device **dev_list = NULL;
	struct ibv_device *ib_dev = NULL;
	int page_size = sysconf(_SC_PAGESIZE);

	/* Check rdma_ctx */
	if(ctx == NULL){
		printf("rdma_ctx is NULL\n");
		return 1;	
	}

	/* Check whether device name is not empty */
	if (!ctx->dev_name) {
		printf("No IB device selected.\n");
		return 1;	
	}

	if(ctx->pkt_size <= 0 || ctx->pkt_cnt <= 0){
		printf("Pakcet size and count is equal to zero or smaller\n");
		return 1;
	}

	/* Get IB device list */
	dev_list = ibv_get_device_list(NULL);
	if (!dev_list) {
		perror("Failed to get IB devices list");
		return 1;
	}

	/* Find desired IB device by its name */
	for (int i = 0; dev_list[i]; ++i){
		if (!strcmp(ibv_get_device_name(dev_list[i]), ctx->dev_name)){
			ib_dev = dev_list[i];
			break;
		}
	}

	/* Check whether device was found or not */
	if (!ib_dev) {
		fprintf(stderr, "IB device %s not found\n", ctx->dev_name);
		goto free_devlist;
	}
	
	/* Calculate sge and buffer size */
	sge_size = (uint64_t)ctx->pkt_size + 40;
	buf_size = sge_size*ctx->pkt_cnt;

	/* Alocate memory buffer where incoming packet will be stores */
	ctx->buf = memalign(page_size, buf_size);
	if (!ctx->buf) {
		fprintf(stderr, "Couldn't allocate work buf.\n");
		goto free_devlist;
	}
	memset(ctx->buf, 0xff,buf_size);

	/* Open infiniband device */
	ctx->context = ibv_open_device(ib_dev);
	if (!ctx->context) {
		fprintf(stderr, "Couldn't get context for %s\n",
			ibv_get_device_name(ib_dev));
		goto clean_buffer;
	}

	/* Check whether active MTU is larger or equal to packet size */
	{

		struct ibv_device_attr dev_attr;
		struct ibv_port_attr port_info = {};
		int mtu;

		if (ibv_query_port(ctx->context, port, &port_info)) {
			fprintf(stderr, "Unable to query port info for port %d\n", port);
			goto clean_device;
		}

		mtu = 1 << (port_info.active_mtu + 7);
		if (ctx->pkt_size > mtu) {
			fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
			goto clean_device;
		}
	}

#ifdef USE_EVENTS
	/* Create completion channel - we are gonna use events */
	ctx->channel = ibv_create_comp_channel(ctx->context);
	if (!ctx->channel) {
		fprintf(stderr, "Couldn't create completion channel\n");
		goto clean_device;
	}
#endif

	/* Allocate protection domain */
	ctx->pd = ibv_alloc_pd(ctx->context);
	if (!ctx->pd) {
		fprintf(stderr, "Couldn't allocate PD\n");
		goto clean_comp_channel;
	}
	/* Register memory region */
	ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, buf_size, IBV_ACCESS_LOCAL_WRITE);
	if (!ctx->mr) {
		fprintf(stderr, "Couldn't register MR\n");
		goto clean_pd;
	}
	/* Prepare scatter gather list using alocated buffer and memory region lkey */
	if((ctx->sge_list = prepare_sge_list(sge_size, ctx->pkt_cnt, ctx->mr->lkey, ctx->buf)) == NULL){
		printf("Couldn't prepare sge list\n");
		goto clean_mr;
	}
	/* Create Compeletion Queue */
	ctx->cq = ibv_create_cq(ctx->context, ctx->rx_depth + 1, NULL,
				ctx->channel, 0);
	if (!ctx->cq) {
		fprintf(stderr, "Couldn't create CQ\n");
		goto clean_sgelist;
	}
	/* Create Queue Pair */
	{
		struct ibv_qp_init_attr init_attr = {
			.send_cq = ctx->cq,
			.recv_cq = ctx->cq,
			.cap     = {
				.max_send_wr  = 1,
				.max_recv_wr  = ctx->rx_depth,
				.max_send_sge = 1,
				.max_recv_sge = 1
			},
			.qp_type = IBV_QPT_UD,
		};

		ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
		if (!ctx->qp)  {
			fprintf(stderr, "Couldn't create QP\n");
			goto clean_cq;
		}
	}

	{
		struct ibv_qp_attr attr = {
			.qp_state        = IBV_QPS_INIT,
			.pkey_index      = 0,
			.port_num        = port,
			.qkey            = ctx->q_key,
		};

		if (ibv_modify_qp(ctx->qp, &attr,
				  IBV_QP_STATE              |
				  IBV_QP_PKEY_INDEX         |
				  IBV_QP_PORT               |
				  IBV_QP_QKEY)) {
			fprintf(stderr, "Failed to modify QP to INIT\n");
			goto clean_qp;
		}
	}
	
	ctx->q_key = CECILIE_QKEY;
	ctx->psn = 0;
	ctx->qp_num = ctx->qp->qp_num;

	ibv_free_device_list(dev_list);

	return 0;

clean_qp:
	ibv_destroy_qp(ctx->qp);

clean_cq:
	ibv_destroy_cq(ctx->cq);

clean_sgelist:
	free(ctx->sge_list);

clean_mr:
	ibv_dereg_mr(ctx->mr);

clean_pd:
	ibv_dealloc_pd(ctx->pd);

clean_comp_channel:
#ifdef USE_EVENTS
	ibv_destroy_comp_channel(ctx->channel);
#endif
clean_device:
	ibv_close_device(ctx->context);

clean_buffer:
	free(ctx->buf);

free_devlist:
	ibv_free_device_list(dev_list);

	return 1;
}


struct ib_payload_t {
        uint8_t gthdr[40];
        uint8_t data[];
};

#if 0

int rdma_transfer(struct cecilie_t *dev, struct rdma_context_t *ctx)
{
	int routs = 0;
	int rcnt = 0;
	int num_cq_events = 0;
	int posted = 0;
	int tmp = 0;
	int ne;	
	struct ibv_cq *ev_cq;
	void *ev_ctx;
	struct ibv_wc wc[100];



	/* Check rdma_cts */
	if(ctx == NULL){
		printf("rdma_ctx is NULL\n");
		return 1;	
	}

	if(ctx->pkt_size <= 0 || ctx->pkt_cnt <= 0){
		printf("Pakcet size and count is equal to zero or smaller\n");
		return 1;
	}

	/* Post receive requests - as many as we can */
	routs = post_rdma_recv(ctx, posted, ctx->pkt_cnt, ctx->rx_depth);
	posted += routs;
	if (routs < 0) {
		fprintf(stderr, "Couldn't post receive (%d)\n", routs);
		return 1;
	}

	/* Set events if we use them - second option is to pool CQ for completion */
	if (ibv_req_notify_cq(ctx->cq, 0)) {
		fprintf(stderr, "Couldn't request CQ notification\n");
		return 1;
	}

	/* Set QP to Ready To Receive state */
	struct ibv_qp_attr attr = {
		.qp_state		= IBV_QPS_RTR
	};

	/* This change QP state to Ready to receive */
	if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
		fprintf(stderr, "Failed to modify QP to RTR\n");
		return 1;
	}

	/* Allow Cecilie to send data */
	write_cecilie_reg(dev, REG_CONTROL, 1);

	while (rcnt < ctx->pkt_cnt) {
		/* Wait for event */
		if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
			fprintf(stderr, "Failed to get cq_event\n");
			return 1;
		}
		++num_cq_events;
		/* Check if event is related to out QP */
		if (ev_cq != ctx->cq) {
			fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
			return 1;
		}
		/* Set notification on next event */
		if (ibv_req_notify_cq(ctx->cq, 0)) {
			fprintf(stderr, "Couldn't request CQ notification\n");
			return 1;
		}
		/* Get completion queue enteries */
		if((ne = ibv_poll_cq(ctx->cq, sizeof(wc)/sizeof(*wc), wc)) < 0) {
			fprintf(stderr, "poll CQ failed %d\n", ne);
			return 1;
		}
		/* Process all completion enteries */
		for (int i = 0; i < ne; ++i) {
			if (wc[i].status != IBV_WC_SUCCESS) {
				fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
					ibv_wc_status_str(wc[i].status),
					wc[i].status, (int) wc[i].wr_id);
				return 1;
			}

			if((int) wc[i].wr_id == RDMA_RECV_WRID){
				if (--routs <= 10) {
					if((tmp = post_rdma_recv(ctx, posted, ctx->pkt_cnt, 100)) < 0){
						fprintf(stderr, "Couldn't post receive (%d)\n", routs);
						return 1;
					}
					posted += tmp;
					routs += tmp;
				}
				++rcnt;
			}else{
				fprintf(stderr, "Completion for unknown wr_id %d\n",
					(int) wc[i].wr_id);
				return 1;
			}
		}
	}

	ibv_ack_cq_events(ctx->cq, num_cq_events);

	/* Stop Cecilie from sending data */
	write_cecilie_reg(dev, REG_CONTROL, 0);

	return 0;
}

#else


int rdma_transfer(struct cecilie_t *dev, struct rdma_context_t *ctx)
{
	int routs = 0;
	int rcnt = 0;
	int num_cq_events = 0;
	int posted = 0;
	int tmp = 0;
	int ne = 0;	
	struct ibv_cq *ev_cq;
	void *ev_ctx;
	struct ibv_wc wc[10000];
	int ret = 0;
	struct timeval start, end, tv;

	/* Check rdma_cts */
	if(ctx == NULL){
		printf("rdma_ctx is NULL\n");
		return 1;	
	}

	if(ctx->pkt_size <= 0 || ctx->pkt_cnt <= 0){
		printf("Pakcet size and count is equal to zero or smaller\n");
		return 1;
	}

	/* Post receive requests - as many as we can */
	routs = post_rdma_recv(ctx, posted, ctx->pkt_cnt, ctx->rx_depth);
	posted += routs;
	if (routs < 0) {
		fprintf(stderr, "Couldn't post receive (%d)\n", routs);
		return 1;
	}

#ifdef USE_EVENTS
	/* Set events if we use them - second option is to pool CQ for completion */
	if (ibv_req_notify_cq(ctx->cq, 0)) {
		fprintf(stderr, "Couldn't request CQ notification\n");
		return 1;
	}
#endif
	/* Set QP to Ready To Receive state */
	struct ibv_qp_attr attr = {
		.qp_state		= IBV_QPS_RTR
	};

	/* This change QP state to Ready to receive */
	if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
		fprintf(stderr, "Failed to modify QP to RTR\n");
		return 1;
	}

#ifdef USE_EVENTS

	int flags;

	/* change the blocking mode of the completion channel */
	flags = fcntl(ctx->channel->fd, F_GETFL);
	if(fcntl(ctx->channel->fd, F_SETFL, flags | O_NONBLOCK) < 0) {
		fprintf(stderr, "Failed to change file descriptor of Completion Event Channel\n");
		return 1;
	}


	struct pollfd my_pollfd;
	int ms_timeout = 5000;

	my_pollfd.fd      = ctx->channel->fd;
	my_pollfd.events  = POLLIN;
	my_pollfd.revents = 0;
#endif
	gettimeofday(&start, NULL);
	/* Allow Cecilie to send data */
	if(write_cecilie_reg(dev, REG_CONTROL, 1))
		return 1;

	while (rcnt < ctx->pkt_cnt) {	

#ifdef USE_EVENTS
		/* Wait for cq event in non blocking mode */
		tmp = poll(&my_pollfd, 1, ms_timeout);
		if(tmp < 0){
			fprintf(stderr, "poll failed\n");
			break;
		}else if(tmp == 0){
			printf("Timeout while waiting for cq event (rcvd %d)\n", rcnt);
			break;
		}
#endif

#ifdef USE_EVENTS
		/* Wait for event - get event */
		if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
			fprintf(stderr, "Failed to get cq_event\n");
			break;
		}
		++num_cq_events;

		/* Set notification on next event */
		if (ibv_req_notify_cq(ctx->cq, 0)) {
			fprintf(stderr, "Couldn't request CQ notification\n");
			break;
		}

		/* Check if event is related to out QP */
		if (ev_cq != ctx->cq) {
			fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
			break;
		}
		/* Get completion queue enteries */
		if((ne = ibv_poll_cq(ctx->cq, sizeof(wc)/sizeof(*wc), wc)) < 0) {
			fprintf(stderr, "poll CQ failed %d\n", ne);
			break;
		}
		
#else
		/* Get completion queue enteries */
		while((ne = ibv_poll_cq(ctx->cq, sizeof(wc)/sizeof(*wc), wc)) == 0) {
			if(ne < 0){
				fprintf(stderr, "poll CQ failed %d\n", ne);
				break;
			}
		}
#endif	
		/* Post new receives */
		if (routs - ne < RDMA_RX_WR_DEPTH/2) {
			if((tmp = post_rdma_recv(ctx, posted, ctx->pkt_cnt, 
					(routs - ne) < 0 ? ctx->rx_depth : ctx->rx_depth - routs - ne)) < 0){
				fprintf(stderr, "Couldn't post receive (%d)\n", routs);
				break;
			}
			posted += tmp;
			routs += tmp;
		}

		/* Process all completion enteries */
		for (int i = 0; i < ne; ++i) {
			if (wc[i].status != IBV_WC_SUCCESS) {
				fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
					ibv_wc_status_str(wc[i].status),
					wc[i].status, (int) wc[i].wr_id);
				goto stop;
			}
	
			if((int) wc[i].wr_id == RDMA_RECV_WRID){
				rcnt++;
				routs--;
			}else{
				fprintf(stderr, "Completion for unknown wr_id %d\n",
					(int) wc[i].wr_id);
				goto stop;
			}
		}
	}

stop:
	/* Capture time */
	gettimeofday(&end, NULL);

	/* Stop Cecilie from sending data */
	write_cecilie_reg(dev, REG_CONTROL, 0);
	ibv_ack_cq_events(ctx->cq, num_cq_events);

	/* If number of received packets is smaller than desired return 1 */
	if(rcnt != ctx->pkt_cnt){
		return 1;
	}

	/* get transfer duration */
	timersub(&end, &start, &tv);
	printf("Transfer time: %ld ms, Transfered: %lf GB, Throughput: %lf GBps / %lf Gbps\n", 
		TIME_MS(tv), (double)((ctx->pkt_size + 40)*ctx->pkt_cnt)/(double)1000000000,
		((double)((ctx->pkt_size + 40)*ctx->pkt_cnt)/(double)1000000000)/((double)(TIME_MS(tv))/(double)1000),
		((double)((ctx->pkt_size + 40)*ctx->pkt_cnt*8)/(double)1000000000)/((double)(TIME_MS(tv))/(double)1000));


	return 0;

}

#endif

int rdma_server(cecilie_t *dev, struct rdma_context_t *ctx)
{
	struct in_addr dst_ip;
	uint8_t dst_mac[8];
	uint32_t dst_udp = CECILIE_ROCE_PORT;
	int ret = 1;
	FILE *fp = NULL;
	char file_path[120];
	struct ib_payload_t *pkt = NULL;
	cecilie_reg_t regs[] = {
		{REG_DST_IP,		0},
		{REG_DST_MAC_HIGH,	0},
		{REG_DST_MAC_LOW,	0},
		{REG_DST_UDP,		0},       
		{REG_DST_QP,		0},        
		{REG_FIRST_PSN,		0},     
		{REG_Q_KEY,		0},         
	};
	
	if(!dev || !ctx)
		return 1;

	memset(dst_mac, 0, sizeof(dst_mac));

	snprintf(file_path, sizeof(file_path), "./results/%s", ctx->file_name);
	if((fp = fopen(file_path, "wb")) == NULL){
		printf("Couldn't open file %s, %s\n", file_path, strerror(errno));
		return 1;
	}

	printf("Setting cecilie registers ...\n");

	/* Stop Cecilie from sending data during initialization */
	write_cecilie_reg(dev, REG_CONTROL, 0);

	/* Get device interface and infiniband device name */
	if(find_rdma_device(dev->adapter_name, ctx->dev_name, sizeof(ctx->dev_name))){
		printf("Interface doesn't support RDMA %s\n", dev->adapter_name);
		fclose(fp);
		return 1;
	}

	/* init rdma receive queue and others things */
	if(init_rdma_ctx(ctx)){
		printf("Couldn't initialize RDMA context\n");
		fclose(fp);
		return 1;
	}

	/* Get interface MAC and IP addresses */
	get_mac_ip(dev->adapter_name, (uint8_t*)dst_mac, &dst_ip);

	/* Set registers in configuration buffer to actual values */
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_DST_IP, htonl(dst_ip.s_addr));
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_DST_MAC_HIGH, dst_mac[0] << 8 | dst_mac[1]);
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_DST_MAC_LOW, htonl(*(uint32_t*)&dst_mac[2]));
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_DST_UDP, dst_udp);
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_DST_QP, ctx->qp_num);
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_FIRST_PSN, ctx->psn);
	set_reg(regs, sizeof(regs)/sizeof(*regs), REG_Q_KEY, ctx->q_key);

	/* Write configuration to Cecilie */
	write_cecilie_config(dev, regs, sizeof(regs)/sizeof(*regs));

	printf("Starting RDMA server (pkt_size [%d], pkt_cnt [%d], rx_depth [%d]) ...\n", ctx->pkt_size, ctx->pkt_cnt, ctx->rx_depth);
	/* Start RDMA transfer */
	if(rdma_transfer(dev, ctx) == 0){
		ret = 0;
		/* Write received data to file */
		for(uint64_t i = 0; i < ctx->pkt_cnt; i++){
			pkt = (struct ib_payload_t *)&ctx->buf[(uint64_t)i*(ctx->pkt_size+40)];
			fwrite(pkt->data, ctx->pkt_size, 1, fp);
		}
	}

	close_rdma_ctx(ctx);
	fclose(fp);

	return ret;
}

int write_menu(cecilie_t *dev)
{
	char opt;
	uint32_t reg_addr;
	uint32_t reg_val;
	char file_name[200];
	cecilie_reg_t *cec_config = NULL;
	int config_size = 0;

	if(!dev)
		return 1;

	printf("Choose 'r' for register or 'c' for configuration from file: ");
	opt = get_opt();

	switch(opt){
		case 'r':
			printf("Register address: ");
			if(scanf("%x", &reg_addr) != 1){
				DISCARD_STDIN;
				printf("Wrong input!\n");
				break;
			}
			printf("Register value: ");
			if(scanf("%x", &reg_val) != 1){
				DISCARD_STDIN;
				printf("Wrong input!\n");
				break;
			}
			printf("Writing value %08x to register %08x\n", reg_val, reg_addr);
			write_cecilie_reg(dev, reg_addr, reg_val);
			break;
		case 'c':
			printf("File name: ");
			if(scanf("%199s", file_name) != 1){
				DISCARD_STDIN;
				printf("Wrong input!\n");
				break;
			}
			printf("Selected file name: %s\n", file_name);
			config_size = file_read_config(file_name, &cec_config);
			if(config_size == 0){
				printf("Nothing was read\n");
				break;
			}
			print_config(cec_config, config_size);
			write_cecilie_config(dev, cec_config, config_size);
			free(cec_config);
			break;
		default:
			break;
	}

	return 0;
}

int read_menu(cecilie_t *dev)
{
	char opt;
	uint32_t reg_addr;
	uint32_t reg_val;
	cecilie_reg_t *cec_config = NULL;
	int config_size = 0;
	char file_name[200];

	if(!dev)
		return 1;

	printf("Choose 'r' for register or 'c' for read to configuraion file: ");
	opt = get_opt();

	switch(opt){
		case 'r':	
			printf("Register address: ");
			if(scanf("%x", &reg_addr) != 1){
				DISCARD_STDIN;
				printf("Wrong input!\n");
				break;
			}
			printf("Reading from register %08x\n", reg_addr);
			read_cecilie_reg(dev, reg_addr, &reg_val);
			break;
		case 'c':
			printf("Registers addresses file name [RO]: ");
			if(scanf("%199s", file_name) != 1){
				DISCARD_STDIN;
				printf("Wrong input!\n");
				break;
			}
			printf("Selected file name: %s\n", file_name);
			config_size = file_read_regs(file_name, &cec_config);
			print_config(cec_config, config_size);
			if(read_cecilie_config(dev, cec_config, config_size)){
				printf("Something went wrong\n");
				break;
			}
			printf("Configuration file name [W]: ");
			if(scanf("%199s", file_name) != 1){
				DISCARD_STDIN;
				printf("Wrong input!\n");
				break;
			}
			file_write_regs(file_name, cec_config, config_size);
			free(cec_config);
			break;
		default:
			printf("Invalid option\n");
			break;
	}
	return 0;
}

int rdma_menu(cecilie_t *dev)
{
	struct rdma_context_t rdma_ctx;

	if(!dev)
		return 1;

	memset(&rdma_ctx, 0, sizeof(rdma_ctx));

	rdma_ctx.q_key = CECILIE_QKEY,
	rdma_ctx.rx_depth = RDMA_RX_WR_DEPTH,

#ifndef DEFAULT_PKT_SIZE
	/* Get size of RDMA packets (data size) */
	printf("Packet size: ");
	if(scanf("%d", &rdma_ctx.pkt_size) != 1){
		DISCARD_STDIN;
		printf("Wrong input!\n");
		return 1;
	}

#else
	/* Get size of RDMA packets (data size) */
	printf("Packet size: default %d\n", DEFAULT_PKT_SIZE);
	rdma_ctx.pkt_size = DEFAULT_PKT_SIZE;
	
#endif

	/* Get number of RDMA packet which should be stored */
	printf("Packet count: ");
	if(scanf("%d", &rdma_ctx.pkt_cnt) != 1){
		DISCARD_STDIN;
		printf("Wrong input!\n");
		return 1;
	}


	/* Get name of file data are gonna be stored to */
	printf("Results file name: ");
	if(scanf("%99s", rdma_ctx.file_name) != 1){
		DISCARD_STDIN;
		printf("Wrong input!\n");
		return 1;
	}
	/* Start rdma server and receive data */
	if(rdma_server(dev, &rdma_ctx)){
		return 1;
	}

	return 0;
}


int main(int argc, char *argv[])
{
	struct cecilie_app_context_t app_ctx;
	char **if_names = NULL;
	int if_num = 0;
	int board_num = 0;
	char opt = '\n';
	int nexit = 1;
	int dev_num;
	memset(&app_ctx, 0, sizeof(app_ctx));

	/* Let user know what this is */
	printf("*** Cecilie control ***\n\n");
	/* Show usage */
	help();
	printf("\n");
	print_register_description();
	printf("\n");

	/* Get adapters */
	if_num = get_adapter_names(&if_names);
	printf("Number of available interfaces: %d\n", if_num);
	print_interfaces(if_names, if_num);
	printf("\n");

	while(nexit){
		printf("Choose option: ");
		opt = get_opt();
		switch(opt){
			case 'h':
				help();
				break;
			case 'd':
				print_register_description();
				break;
			case 'f':
				printf("Looking for Cecilies ...\n");
				clear_cecilie_list(&app_ctx.dev_list);
				for(int i = 0; i < if_num; i++)
				{
					printf("\t");
					board_num += find_cecilie(if_names[i], &app_ctx.dev_list);
				}
				if(board_num <= 0){
					printf("No board found\n");
					break;
				}
				print_cecilie_list(app_ctx.dev_list);
				printf("Choose board: ");
				if(scanf("%d", &dev_num) != 1){
					DISCARD_STDIN;
					printf("Wrong input [%d]!\n", dev_num);
					app_ctx.dev = NULL;
					break;
				}
				app_ctx.dev = get_cecilie_from_list(app_ctx.dev_list, dev_num);
				if(app_ctx.dev == NULL)
					printf("Non existing board!\n");
				print_cecilie(app_ctx.dev);
				break;
			case 'l':
				printf("Print Cecilie list\n");
				print_cecilie_list(app_ctx.dev_list);
				break;
			case 'c':
				printf("Choose board from list\n");
				print_cecilie_list(app_ctx.dev_list);
				printf("Choose board number: ");
				if(scanf("%d", &dev_num) != 1){
					DISCARD_STDIN;
					printf("Wrong input [%d]!\n", dev_num);
					break;
				}
				app_ctx.dev = get_cecilie_from_list(app_ctx.dev_list, dev_num);
				if(app_ctx.dev == NULL)
					printf("Non existing board!\n");
				print_cecilie(app_ctx.dev);
				break;
			case 'w':
				printf("Write to Cecilie register\n");
				if(!app_ctx.dev){
					printf("no board chosen, run 'c' command\n");
					break;
				}
				write_menu(app_ctx.dev);
				break;
			case 'r':
				printf("Read from Cecilie register\n");		
				if(!app_ctx.dev){
					printf("no board chosen, run 'c' command\n");
					break;
				}
				read_menu(app_ctx.dev);
				break;
			case 't':
				printf("Start RDMA transfer\n");
				if(!app_ctx.dev){
					printf("no board chosen, run 'c' command\n");
					break;
				}
				if(rdma_menu(app_ctx.dev)){
					printf("Something went wrong\n");
				}
				break;
			case 'e':
				printf("Exit\n");
				nexit = 0;
				break;
			default:
				printf("Invalid option\n");
			       	break;
		}
		printf("\n");
	}

	return 0;
}
