libev多线程使用例

libev多线程使用例，参考memcached使用方法。

#include
#include
#include
#include
#include
#include
#include
#include
#include
#define MAXLEN 1023
#define PORT 9999
#define ADDR_IP "192.168.206.129"


int socket_init();
void accept_callback(struct ev_loop *loop, ev_io *w, int revents);
void recv_callback(struct ev_loop *loop, ev_io *w, int revents);
void write_callback(struct ev_loop *loop, ev_io *w, int revents);


/* An item in the connection queue. */
typedef struct conn_queue_item CQ_ITEM;
struct conn_queue_item {
    int               sfd;
    char szAddr[16];
    int port;
    CQ_ITEM          *next;
};


/* A connection queue. */
typedef struct conn_queue CQ;
struct conn_queue {
    CQ_ITEM *head;
    CQ_ITEM *tail;
    pthread_mutex_t lock;
};


/*
 * Initializes a connection queue.
 */
static void cq_init(CQ *cq) {
    pthread_mutex_init(&cq->lock, NULL);
    cq->head = NULL;
    cq->tail = NULL;
}


/*
 * Looks for an item on a connection queue, but doesn't block if there isn't
 * one.
 * Returns the item, or NULL if no item is available
 */
static CQ_ITEM *cq_pop(CQ *cq) {
    CQ_ITEM *item;


    pthread_mutex_lock(&cq->lock);
    item = cq->head;
    if (NULL != item) {
        cq->head = item->next;
        if (NULL == cq->head)
            cq->tail = NULL;
    }
    pthread_mutex_unlock(&cq->lock);


    return item;
}


/*
 * Adds an item to a connection queue.
 */
static void cq_push(CQ *cq, CQ_ITEM *item) {
    item->next = NULL;


    pthread_mutex_lock(&cq->lock);
    if (NULL == cq->tail)
        cq->head = item;
    else
        cq->tail->next = item;
    cq->tail = item;
    pthread_mutex_unlock(&cq->lock);
}




typedef struct {
    pthread_t thread_id;         /* unique ID of this thread */
    struct ev_loop *loop;     /* libev loop this thread uses */
    struct ev_async async_watcher;   /* async watcher for new connect */
    struct conn_queue *new_conn_queue; /* queue of new connections to handle */
} WORK_THREAD;


typedef struct {
    pthread_t thread_id;         /* unique ID of this thread */
    struct ev_loop *loop;     /* libev loop this thread uses */
    struct ev_io accept_watcher;   /* accept watcher for new connect */
} DISPATCHER_THREAD;


/*
 * libev default loop, with a accept_watcher to accept the new connect
 * and dispatch to WORK_THREAD.
 */
static DISPATCHER_THREAD dispatcher_thread;
/*
 * Each libev instance has a async_watcher, which other threads
 * can use to signal that they've put a new connection on its queue.
 */
static WORK_THREAD *work_threads;
/*
 * Number of worker threads that have finished setting themselves up.
 */
static int init_count = 0;
static pthread_mutex_t init_lock;
static pthread_cond_t init_cond;
static int round_robin = 0;


/*
 * Worker thread: main event loop
 */
static void *worker_libev(void *arg) {
    WORK_THREAD *me = arg;


    /* Any per-thread setup can happen here; thread_init() will block until
     * all threads have finished initializing.
     */


    pthread_mutex_lock(&init_lock);
    init_count++;
    pthread_cond_signal(&init_cond);
    pthread_mutex_unlock(&init_lock);


me->thread_id = pthread_self();
    ev_loop(me->loop, 0);
    return NULL;
}


/*
 * Creates a worker thread.
 */
static void create_worker(void *(*func)(void *), void *arg) {
    pthread_t       thread;
    pthread_attr_t  attr;
    int             ret;


    pthread_attr_init(&attr);


    if ((ret = pthread_create(&thread, &attr, func, arg)) != 0) {
        fprintf(stderr, "Can't create thread: %s\n",
                strerror(ret));
        exit(1);
    }
}


static void
async_cb (EV_P_ ev_async *w, int revents)
{
    CQ_ITEM *item;
    
    item = cq_pop(((WORK_THREAD*)(w->data))->new_conn_queue);


    if (NULL != item) {
        ev_io* recv_watcher=malloc(sizeof(ev_io));
        ev_io_init(recv_watcher,recv_callback,item->sfd,EV_READ);
        ev_io_start(((WORK_THREAD*)(w->data))->loop,recv_watcher);
 
        printf("thread[%lu] accept: fd :%d  addr:%s port:%d\n",((WORK_THREAD*)(w->data))->thread_id,item->sfd,item->szAddr,item->port);
 
        //cqi_free(item);
        free(item);
        item = NULL;
    }
}


/*
 * Set up a thread's information.
 */
static void setup_thread(WORK_THREAD *me) {
    me->loop = ev_loop_new(0);
    if (! me->loop) {
        fprintf(stderr, "Can't allocate event base\n");
        exit(1);
    }


me->async_watcher.data = me;
    /* Listen for notifications from other threads */
    ev_async_init(&me->async_watcher, async_cb);
  ev_async_start(me->loop, &me->async_watcher);


    me->new_conn_queue = malloc(sizeof(struct conn_queue));
    if (me->new_conn_queue == NULL) {
        perror("Failed to allocate memory for connection queue\n");
        exit(EXIT_FAILURE);
    }
    cq_init(me->new_conn_queue);
}


void thread_init()
{
    int nthreads = 4;
 pthread_mutex_init(&init_lock, NULL);
    pthread_cond_init(&init_cond, NULL);
    work_threads = calloc(nthreads, sizeof(WORK_THREAD));
    if (! work_threads) {
        perror("Can't allocate thread descriptors\n");
        exit(1);
    }


    //dispatcher_thread.loop = EV_DEFAULT_UC;
    //dispatcher_thread.thread_id = pthread_self();
    dispatcher_thread.loop = ev_default_loop(0);
  dispatcher_thread.thread_id = pthread_self();
    
    int i = 0;
    for (i = 0; i < nthreads; i++) {
        setup_thread(&work_threads[i]);
    }


    /* Create threads after we've done all the libevent setup. */
    for (i = 0; i < nthreads; i++) {
        create_worker(worker_libev, &work_threads[i]);
    }


    /* Wait for all the threads to set themselves up before returning. */
    pthread_mutex_lock(&init_lock);
    while (init_count < nthreads) {
        pthread_cond_wait(&init_cond, &init_lock);
    }
    pthread_mutex_unlock(&init_lock);
}

int main(int argc ,char** argv)
{
 thread_init();
 int listen;
 listen=socket_init();
 ev_io_init(&(dispatcher_thread.accept_watcher), accept_callback,listen, EV_READ);
 ev_io_start(dispatcher_thread.loop,&(dispatcher_thread.accept_watcher)); 
 ev_loop(dispatcher_thread.loop,0);
 ev_loop_destroy(dispatcher_thread.loop);
 return 0;
}

int socket_init()
{
 struct sockaddr_in my_addr;
 int listener;
 if ((listener = socket(AF_INET, SOCK_STREAM, 0)) == -1) 
 {
  perror("socket");
  exit(1);
 } 
 else
 {
  printf("SOCKET CREATE SUCCESS! \n");
 }
 //setnonblocking(listener);
 int so_reuseaddr=1;
 setsockopt(listener,SOL_SOCKET,SO_REUSEADDR,&so_reuseaddr,sizeof(so_reuseaddr));
 bzero(&my_addr, sizeof(my_addr));
 my_addr.sin_family = PF_INET;
 my_addr.sin_port = htons(PORT);
 my_addr.sin_addr.s_addr = inet_addr(ADDR_IP);


 if (bind(listener, (struct sockaddr *) &my_addr, sizeof(struct sockaddr))== -1) 
 {
  perror("bind error!\n");
  exit(1);
 } 
 else
 {
  printf("IP BIND SUCCESS,IP:%s\n",ADDR_IP);
 }


 if (listen(listener, 1024) == -1) 
 {
  perror("listen error!\n");
  exit(1);
 } 
 else
 {
  printf("LISTEN SUCCESS,PORT:%d\n",PORT);
 }
 return listener;
}


void dispath_conn(int anewfd,struct sockaddr_in asin)
{
// set the new connect item
CQ_ITEM *lpNewItem = calloc(1,sizeof(CQ_ITEM));
if (! lpNewItem) {
     perror("Can't allocate connection item\n");
     exit(1);
  }
  
  lpNewItem->sfd = anewfd;
  strcpy(lpNewItem->szAddr,(char*)inet_ntoa(asin.sin_addr));
  lpNewItem->port = asin.sin_port;


// libev default loop, accept the new connection, round-robin 
// dispath to a work_thread.
  int robin = round_robin%init_count;
  cq_push(work_threads[robin].new_conn_queue,lpNewItem);
  ev_async_send(work_threads[robin].loop, &(work_threads[robin].async_watcher));
  round_robin++;
} 


void accept_callback(struct ev_loop *loop, ev_io *w, int revents)
{
 int newfd;
 struct sockaddr_in sin;
 socklen_t addrlen = sizeof(struct sockaddr);
 while ((newfd = accept(w->fd, (struct sockaddr *)&sin, &addrlen)) < 0)
 {
  if (errno == EAGAIN || errno == EWOULDBLOCK) 
  {
   //these are transient, so don't log anything.
   continue; 
  }
  else
  {
   printf("accept error.[%s]\n", strerror(errno));
   break;
  }
 }
 //ev_io* accept_watcher=malloc(sizeof(ev_io));
 //ev_io_init(accept_watcher,recv_callback,newfd,EV_READ);
 //ev_io_start(loop,accept_watcher);
 
 //printf("accept callback : fd :%d  addr:%s port:%d\n",accept_watcher->fd,(char*)inet_ntoa(sin.sin_addr),sin.sin_port);
 dispath_conn(newfd,sin);
 
}


void recv_callback(struct ev_loop *loop, ev_io *w, int revents)
{
 char buffer[1024]={0};
 int ret =0;
 //ev_io write_event;
loop:
 ret=recv(w->fd,buffer,MAXLEN,0);
 if(ret > 0)
 {
  printf("thread[%lu] recv message :%s  \n",pthread_self(),buffer);
  
 }
 else if(ret ==0)
 {
  printf("thread[%lu] remote socket closed!socket fd: %d\n",pthread_self(),w->fd);
  close(w->fd);
  ev_io_stop(loop, w);
  free(w);
  return;
 }
 else
 {
  if(errno == EAGAIN ||errno == EWOULDBLOCK)
  {
    goto loop;
  }
  else
  {
   printf("thread[%lu] ret :%d ,close socket fd : %d\n",pthread_self(),ret,w->fd);
   close(w->fd);
   ev_io_stop(loop, w);
   free(w);
   return;
  }
 }
 int fd=w->fd;
 ev_io_stop(loop,  w);
 ev_io_init(w,write_callback,fd,EV_WRITE);
 ev_io_start(loop,w);
 printf("thread[%lu] socket fd : %d, turn read 2 write loop! ",pthread_self(),fd);
 
}




void write_callback(struct ev_loop *loop, ev_io *w, int revents)
{
 char buffer[1024]={0};
 //ev_io read_event;
 snprintf(buffer,1023,"this is a test!");
 write(w->fd,buffer,strlen(buffer),0);
 int fd=w->fd;
 ev_io_stop(loop,  w);
 ev_io_init(w,recv_callback,fd,EV_READ);
 ev_io_start(loop,w);
}