记LWIP调试http server的Out of memory问题

最近在做IOT控制，主要通过LWIP的http server来做控制，实现手机和电脑浏览器控制查看数据，其中用web server做实时的数据传输，遇到了切换网页是有时会卡在跳转处很久，有时会直接跳转失败，只能重新进入web，于是打开LWIP的调试发现卡死时输出Out of memory的输出，于是我就重新过了变LWIP，看看是什么原因导致的。
下面我们来看LWIP的HTTPD的源码。

void
httpd_init(void)
{
#if HTTPD_USE_MEM_POOL
  LWIP_ASSERT("memp_sizes[MEMP_HTTPD_STATE] >= sizeof(http_state)",
     memp_sizes[MEMP_HTTPD_STATE] >= sizeof(http_state));
  LWIP_ASSERT("memp_sizes[MEMP_HTTPD_SSI_STATE] >= sizeof(http_ssi_state)",
     memp_sizes[MEMP_HTTPD_SSI_STATE] >= sizeof(http_ssi_state));
#endif
  LWIP_DEBUGF(HTTPD_DEBUG, ("httpd_init\n"));

  httpd_init_addr(IP_ADDR_ANY);
}

先判断是否使用内存池分配内存，在进行初始化，本周就是开启一个TCP连接，和SOCKET建立的TCP类似，

static void
httpd_init_addr(ip_addr_t *local_addr)
{
  struct tcp_pcb *pcb;
  err_t err;

  pcb = tcp_new();
  LWIP_ASSERT("httpd_init: tcp_new failed", pcb != NULL);
  tcp_setprio(pcb, HTTPD_TCP_PRIO);
  /* set SOF_REUSEADDR here to explicitly bind httpd to multiple interfaces */
  err = tcp_bind(pcb, local_addr, HTTPD_SERVER_PORT);
  LWIP_ASSERT("httpd_init: tcp_bind failed", err == ERR_OK);
  pcb = tcp_listen(pcb);
  LWIP_ASSERT("httpd_init: tcp_listen failed", pcb != NULL);
  /* initialize callback arg and accept callback */
  tcp_arg(pcb, pcb);
  tcp_accept(pcb, http_accept);
}

这里面做TCP的建立，写个socket的应该都比较熟悉，先bind、在listen、然后accept，只是这里是accept回调函数，在LWIP里面SOCKET也是用回调函数实现的，在linux里面是使用system call来实现socket调用的，我在看下accept在什么时候被回调执行。

#define TCP_EVENT_ACCEPT(pcb,err,ret)                          \
  do {                                                         \
    if((pcb)->accept != NULL)                                  \
      (ret) = (pcb)->accept((pcb)->callback_arg,(pcb),(err));  \
    else (ret) = ERR_ARG;                                      \
  } while (0)

在TCP_imple.h里面实现了多种event事件，用于实现回调，看下那个代码调用了accept事件。

static err_t
tcp_process(struct tcp_pcb *pcb)
{
  struct tcp_seg *rseg;
  u8_t acceptable = 0;
  err_t err;

  err = ERR_OK;

  XXX
#if LWIP_CALLBACK_API
        LWIP_ASSERT("pcb->accept != NULL", pcb->accept != NULL);
#endif
        /* Call the accept function. */
        TCP_EVENT_ACCEPT(pcb, ERR_OK, err);
  XXX
  return ERR_OK;
}

在这个process函数里面其实做了很多事情，不止调用了accept函数，还有很多连接等信息，基本所有的TCP包都要经过他的处理，里面用状态机写的，大家可以看下源码，所以这个函数肯定是被TCP接口函数调用的。

void
tcp_input(struct pbuf *p, struct netif *inp){
    /* If there is data which was previously "refused" by upper layer */
    if (pcb->refused_data != NULL) {
      if ((tcp_process_refused_data(pcb) == ERR_ABRT) ||
        ((pcb->refused_data != NULL) && (tcplen > 0))) {
        /* pcb has been aborted or refused data is still refused and the new
           segment contains data */
        TCP_STATS_INC(tcp.drop);
        snmp_inc_tcpinerrs();
        goto aborted;
      }
    }
    tcp_input_pcb = pcb;
    err = tcp_process(pcb);
   }

这个函数也异常的长，主要是对tcp的各种包做解析，在做处理，比如是不是broadcast的包等等，

err_t
ip_input(struct pbuf *p, struct netif *inp){
#endif /* LWIP_UDP */
#if LWIP_TCP
    case IP_PROTO_TCP:
      snmp_inc_ipindelivers();
      tcp_input(p, inp);
      break;
#endif /* LWIP_TCP */
}

TCP处理肯定是经过ip包传上来的，所以ip_put就是做这个操作的，里面也是用状态机写的，包括了各种类型的包解析，比如udp，icmp等等，ip包要经过arp解析后再传输的，这个是另个内容了，我这边是使用freertos，所以ip包经过一个线程来实时接收。

static void
tcpip_thread(void *arg)
{
  struct tcpip_msg *msg;
  LWIP_UNUSED_ARG(arg);

  if (tcpip_init_done != NULL) {
    tcpip_init_done(tcpip_init_done_arg);
  }

  LOCK_TCPIP_CORE();
  while (1) {                          /* MAIN Loop */
    UNLOCK_TCPIP_CORE();
    LWIP_TCPIP_THREAD_ALIVE();
    /* wait for a message, timeouts are processed while waiting */
    sys_timeouts_mbox_fetch(&mbox, (void **)&msg);
    LOCK_TCPIP_CORE();
    switch (msg->type) {
#if LWIP_NETCONN
    case TCPIP_MSG_API:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API message %p\n", (void *)msg));
      msg->msg.apimsg->function(&(msg->msg.apimsg->msg));
      break;
#endif /* LWIP_NETCONN */

#if !LWIP_TCPIP_CORE_LOCKING_INPUT
    case TCPIP_MSG_INPKT:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: PACKET %p\n", (void *)msg));

      if(msg->msg.inp.p != NULL && msg->msg.inp.netif != NULL) {
#if LWIP_ETHERNET
        if (msg->msg.inp.netif->flags & (NETIF_FLAG_ETHARP | NETIF_FLAG_ETHERNET)) {
          ethernet_input(msg->msg.inp.p, msg->msg.inp.netif);
        } else
#endif /* LWIP_ETHERNET */
#if LWIP_IPV6
        if ((*((unsigned char *)(msg->msg.inp.p->payload)) & 0xf0) == 0x60) {
          ip6_input(msg->msg.inp.p, msg->msg.inp.netif);
        } else
#endif /* LWIP_IPV6 */
        {
          ip_input(msg->msg.inp.p, msg->msg.inp.netif);
        }
      }
      memp_free(MEMP_TCPIP_MSG_INPKT, msg);
      break;
#endif /* LWIP_TCPIP_CORE_LOCKING_INPUT */

#if LWIP_NETIF_API
    case TCPIP_MSG_NETIFAPI:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: Netif API message %p\n", (void *)msg));
      msg->msg.netifapimsg->function(&(msg->msg.netifapimsg->msg));
      break;
#endif /* LWIP_NETIF_API */

#if LWIP_TCPIP_TIMEOUT
    case TCPIP_MSG_TIMEOUT:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: TIMEOUT %p\n", (void *)msg));
      sys_timeout(msg->msg.tmo.msecs, msg->msg.tmo.h, msg->msg.tmo.arg);
      memp_free(MEMP_TCPIP_MSG_API, msg);
      break;
    case TCPIP_MSG_UNTIMEOUT:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: UNTIMEOUT %p\n", (void *)msg));
      sys_untimeout(msg->msg.tmo.h, msg->msg.tmo.arg);
      memp_free(MEMP_TCPIP_MSG_API, msg);
      break;
#endif /* LWIP_TCPIP_TIMEOUT */

    case TCPIP_MSG_CALLBACK:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK %p\n", (void *)msg));
      msg->msg.cb.function(msg->msg.cb.ctx);
      memp_free(MEMP_TCPIP_MSG_API, msg);
      break;

    case TCPIP_MSG_CALLBACK_STATIC:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK_STATIC %p\n", (void *)msg));
      msg->msg.cb.function(msg->msg.cb.ctx);
      break;

    default:
      LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: invalid message: %d\n", msg->type));
      LWIP_ASSERT("tcpip_thread: invalid message", 0);
      break;
    }
  }
}

其中用了一个mbox来确定是否数据接收到了，接收到了就进行解析，如果是IP包者送到ip_input函数去，数据传输最终调用

void
ethernetif_input(struct netif *netif, struct pbuf *p)
{
  struct ethernetif *ethernetif;
  struct eth_hdr *ethhdr;

  if(p == NULL)
    goto _exit;

  if(p->payload == NULL) {
    pbuf_free(p);
    goto _exit;
  }

  if(netif == NULL) {
    goto _exit;
  }

  if (!(netif->flags & NETIF_FLAG_LINK_UP)) {
    pbuf_free(p);
    p = NULL;
    goto _exit;
  }

  ethernetif = netif->state;

  /* points to packet payload, which starts with an Ethernet header */
  ethhdr = p->payload;

  switch (htons(ethhdr->type)) {
  /* IP or ARP packet? */
  case ETHTYPE_IP:
  case ETHTYPE_IPV6:
  case ETHTYPE_ARP:
#if PPPOE_SUPPORT
  /* PPPoE packet? */
  case ETHTYPE_PPPOEDISC:
  case ETHTYPE_PPPOE:
#endif /* PPPOE_SUPPORT */
    /* full packet send to tcpip_thread to process */
    if (netif->input(p, netif)!=ERR_OK)
     { LWIP_DEBUGF(NETIF_DEBUG, ("ethernetif_input: IP input error\n"));
       pbuf_free(p);
       p = NULL;
     }
    break;

  default:
    pbuf_free(p);
    p = NULL;
    break;
  }
_exit:
;
}

可以看到netif->input(p, netif)也是回调函数，这个函数被厂商封装了，最终通过这个接口释放mbox，将数据写入pbuf里面。
到这里一条完整的线路已经很清晰了，http的接收和发送时类似的，我们现在看下接收函数，
当数据到来，且最终accept被调用了

static err_t
http_accept(void *arg, struct tcp_pcb *pcb, err_t err)
{
  struct http_state *hs;
  struct tcp_pcb_listen *lpcb = (struct tcp_pcb_listen*)arg;
  LWIP_UNUSED_ARG(err);
  LWIP_DEBUGF(HTTPD_DEBUG, ("http_accept %p / %p\n", (void*)pcb, arg));

  /* Decrease the listen backlog counter */
  tcp_accepted(lpcb);
  /* Set priority */
  tcp_setprio(pcb, HTTPD_TCP_PRIO);

  /* Allocate memory for the structure that holds the state of the
     connection - initialized by that function. */
  hs = http_state_alloc();
  if (hs == NULL) {
    LWIP_DEBUGF(HTTPD_DEBUG, ("http_accept: Out of memory, RST\n"));
    return ERR_MEM;
  }
  hs->pcb = pcb;

  /* Tell TCP that this is the structure we wish to be passed for our
     callbacks. */
  tcp_arg(pcb, hs);

  /* Set up the various callback functions */
  tcp_recv(pcb, http_recv);
  tcp_err(pcb, http_err);
  tcp_poll(pcb, http_poll, HTTPD_POLL_INTERVAL);
  tcp_sent(pcb, http_sent);

  return ERR_OK;
}

这里面注册了接收很发送的回调函数，POLL函数是重点，本次问题的解决方法就是对POLL进行操作的，rev和send也是在TCP_IMPL.h里面定义了事件

#define TCP_EVENT_SENT(pcb,space,ret)                          \
  do {                                                         \
    if((pcb)->sent != NULL)                                    \
      (ret) = (pcb)->sent((pcb)->callback_arg,(pcb),(space));  \
    else (ret) = ERR_OK;                                       \
  } while (0)

#define TCP_EVENT_RECV(pcb,p,err,ret)                          \
  do {                                                         \
    if((pcb)->recv != NULL) {                                  \
      (ret) = (pcb)->recv((pcb)->callback_arg,(pcb),(p),(err));\
    } else {                                                   \
      (ret) = tcp_recv_null(NULL, (pcb), (p), (err));          \
    }                                                          \
  } while (0)

看下rev在哪里被回调，想想应该也知道是在tcp_input里面调用，http是基于TCP的，所以通过TCP在push到上层去，我们看下tcp_input

        if (recv_data != NULL) {
          LWIP_ASSERT("pcb->refused_data == NULL", pcb->refused_data == NULL);
          if (pcb->flags & TF_RXCLOSED) {
            /* received data although already closed -> abort (send RST) to
               notify the remote host that not all data has been processed */
            pbuf_free(recv_data);
            tcp_abort(pcb);
            goto aborted;
          }

          /* Notify application that data has been received. */
          TCP_EVENT_RECV(pcb, recv_data, ERR_OK, err);
          if (err == ERR_ABRT) {
            goto aborted;
          }

重注释也能看出是从这里调用给上层http使用的，我们看下http rev对数据做了什么处理

static err_t
http_recv(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err){
XXXXXXX
#if LWIP_HTTPD_SUPPORT_POST
  if (hs->post_content_len_left > 0) {
    /* reset idle counter when POST data is received */
    hs->retries = 0;
    /* this is data for a POST, pass the complete pbuf to the application */
    http_post_rxpbuf(hs, p);
    /* pbuf is passed to the application, don't free it! */
    if (hs->post_content_len_left == 0) {
      /* all data received, send response or close connection */
      http_send(pcb, hs);
    }
    return ERR_OK;
  } else
#endif /* LWIP_HTTPD_SUPPORT_POST */
  {
    if (hs->handle == NULL) {
      parsed = http_parse_request(&p, hs, pcb);
      LWIP_ASSERT("http_parse_request: unexpected return value", parsed == ERR_OK
        || parsed == ERR_INPROGRESS ||parsed == ERR_ARG
        || parsed == ERR_USE || parsed == ERR_MEM);
    } else {
      LWIP_DEBUGF(HTTPD_DEBUG, ("http_recv: already sending data\n"));
    }
#if LWIP_HTTPD_SUPPORT_REQUESTLIST
    if (parsed != ERR_INPROGRESS) {
      /* request fully parsed or error */
      if (hs->req != NULL) {
        pbuf_free(hs->req);
        hs->req = NULL;
      }
    }
#else /* LWIP_HTTPD_SUPPORT_REQUESTLIST */
    if (p != NULL) {
      /* pbuf not passed to application, free it now */
      pbuf_free(p);
    }
#endif /* LWIP_HTTPD_SUPPORT_REQUESTLIST */
    if (parsed == ERR_OK) {
#if LWIP_HTTPD_SUPPORT_POST
      if (hs->post_content_len_left == 0)
#endif /* LWIP_HTTPD_SUPPORT_POST */
      {
        LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: data %p len %"S32_F"\n", hs->file, hs->left));
        http_send(pcb, hs);
      }
    } else if (parsed == ERR_ARG || parsed == ERR_MEM) {
      /* @todo: close on ERR_USE? */
      http_close_conn(pcb, hs);
    }
 XXXXXXXXXXXXXXXXXX
 }

rev的函数也很长，我这里截取了重点，其他都是对数据处理的和判断的，这里先判断是POST还是GET的方法，其实http还有其他几个方法，只是不常有这里也没有实现，解析主要在parsed = http_parse_request(&p, hs, pcb);这个函数里面，在这里面会对数据进行判断，看是否调用了html，js，css等文件，这里的这些文件已经被处理成数组了，总之最终查找到的话就会将数据填充到这个结构体里

struct http_state {
#if LWIP_HTTPD_KILL_OLD_ON_CONNECTIONS_EXCEEDED
  struct http_state *next;
#endif /* LWIP_HTTPD_KILL_OLD_ON_CONNECTIONS_EXCEEDED */
  struct fs_file file_handle;
  struct fs_file *handle;
  char *file;       /* Pointer to first unsent byte in buf. */

  u8_t is_websocket;

  struct tcp_pcb *pcb;
#if LWIP_HTTPD_SUPPORT_REQUESTLIST
  struct pbuf *req;
#endif /* LWIP_HTTPD_SUPPORT_REQUESTLIST */

#if LWIP_HTTPD_DYNAMIC_FILE_READ
  char *buf;        /* File read buffer. */
  int buf_len;      /* Size of file read buffer, buf. */
#endif /* LWIP_HTTPD_DYNAMIC_FILE_READ */
  u32_t left;       /* Number of unsent bytes in buf. */
  u8_t retries;
#if LWIP_HTTPD_SUPPORT_11_KEEPALIVE
  u8_t keepalive;
#endif /* LWIP_HTTPD_SUPPORT_11_KEEPALIVE */
#if LWIP_HTTPD_SSI
  struct http_ssi_state *ssi;
#endif /* LWIP_HTTPD_SSI */
#if LWIP_HTTPD_CGI
  char *params[LWIP_HTTPD_MAX_CGI_PARAMETERS]; /* Params extracted from the request URI */
  char *param_vals[LWIP_HTTPD_MAX_CGI_PARAMETERS]; /* Values for each extracted param */
#endif /* LWIP_HTTPD_CGI */
#if LWIP_HTTPD_DYNAMIC_HEADERS
  const char *hdrs[NUM_FILE_HDR_STRINGS]; /* HTTP headers to be sent. */
  u16_t hdr_pos;     /* The position of the first unsent header byte in the
                        current string */
  u16_t hdr_index;   /* The index of the hdr string currently being sent. */
#endif /* LWIP_HTTPD_DYNAMIC_HEADERS */
#if LWIP_HTTPD_TIMING
  u32_t time_started;
#endif /* LWIP_HTTPD_TIMING */
#if LWIP_HTTPD_SUPPORT_POST
  u32_t post_content_len_left;
#if LWIP_HTTPD_POST_MANUAL_WND
  u32_t unrecved_bytes;
  u8_t no_auto_wnd;
  u8_t post_finished;
#endif /* LWIP_HTTPD_POST_MANUAL_WND */
#endif /* LWIP_HTTPD_SUPPORT_POST*/
};

最后调用http_send(pcb, hs);这个函数，这个函数我们在看下，是整个发送的关键

static u8_t
http_send(struct tcp_pcb *pcb, struct http_state *hs)
{
  u8_t data_to_send = HTTP_NO_DATA_TO_SEND;

  LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_send: pcb=%p hs=%p left=%d\n", (void*)pcb,
    (void*)hs, hs != NULL ? (int)hs->left : 0));

#if LWIP_HTTPD_SUPPORT_POST && LWIP_HTTPD_POST_MANUAL_WND
  if (hs->unrecved_bytes != 0) {
    return 0;
  }
#endif /* LWIP_HTTPD_SUPPORT_POST && LWIP_HTTPD_POST_MANUAL_WND */

  /* If we were passed a NULL state structure pointer, ignore the call. */
  if (hs == NULL) {
    return 0;
  }

#if LWIP_HTTPD_FS_ASYNC_READ
  /* Check if we are allowed to read from this file.
     (e.g. SSI might want to delay sending until data is available) */
  if (!fs_is_file_ready(hs->handle, http_continue, hs)) {
    return 0;
  }
#endif /* LWIP_HTTPD_FS_ASYNC_READ */

#if LWIP_HTTPD_DYNAMIC_HEADERS
  /* Do we have any more header data to send for this file? */
  if(hs->hdr_index < NUM_FILE_HDR_STRINGS) {
    data_to_send = http_send_headers(pcb, hs);
    if (data_to_send != HTTP_DATA_TO_SEND_CONTINUE) {
      return data_to_send;
    }
  }
#endif /* LWIP_HTTPD_DYNAMIC_HEADERS */

  /* Have we run out of file data to send? If so, we need to read the next
   * block from the file. */
  if (hs->left == 0) {
    if (!http_check_eof(pcb, hs)) {
      return 0;
    }
  }

#if LWIP_HTTPD_SSI
  if(hs->ssi) {
    data_to_send = http_send_data_ssi(pcb, hs);
  } else
#endif /* LWIP_HTTPD_SSI */
  {
    data_to_send = http_send_data_nonssi(pcb, hs);
  }

  if((hs->left == 0) && (fs_bytes_left(hs->handle) <= 0)) {
    /* We reached the end of the file so this request is done.
     * This adds the FIN flag right into the last data segment. */
    LWIP_DEBUGF(HTTPD_DEBUG, ("End of file.\n"));
    http_eof(pcb, hs);
    return 0;
  }
  LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("send_data end.\n"));
  return data_to_send;
}

这里先做一些判断，因为我这边只开启了SSI，所以最终会调用data_to_send = http_send_data_ssi(pcb, hs);这个函数，进入这个函数

err = http_write(pcb, hs->file, &len, HTTP_IS_DATA_VOLATILE(hs));

里面一大段对SSI的处理，我对SSI不是很熟悉，所以跳过，直接看调用的发送函数

static err_t
http_write(struct tcp_pcb *pcb, const void* ptr, u16_t *length, u8_t apiflags)
{
   u16_t len;
   err_t err;
   LWIP_ASSERT("length != NULL", length != NULL);
   len = *length;
   if (len == 0) {
     return ERR_OK;
   }
   do {
     LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("Trying to send %d bytes\n", len));
     err = tcp_write(pcb, ptr, len, apiflags);
     if (err == ERR_MEM) {
       if ((tcp_sndbuf(pcb) == 0) ||
           (tcp_sndqueuelen(pcb) >= TCP_SND_QUEUELEN)) {
         /* no need to try smaller sizes */
         len = 1;
       } else {
         len /= 2;
       }
       LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE,
                   ("Send failed, trying less (%d bytes)\n", len));
     }
   } while ((err == ERR_MEM) && (len > 1));

   if (err == ERR_OK) {
     LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("Sent %d bytes\n", len));
   } else {
     LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("Send failed with err %d (\"%s\")\n", err, lwip_strerr(err)));
   }

   *length = len;
   return err;
}

这里发送的时候会对分配到的内存是否足够进行判断，如果不够就会先减少发送的大小，在循环检测直到发送完毕，在看下tcp_write这个函数，我们看下注释
/**
* Write data for sending (but does not send it immediately).
*
* It waits in the expectation of more data being sent soon (as
* it can send them more efficiently by combining them together).
* To prompt the system to send data now, call tcp_output() after
* calling tcp_write().
*
* @param pcb Protocol control block for the TCP connection to enqueue data for.
* @param arg Pointer to the data to be enqueued for sending.
* @param len Data length in bytes
* @param apiflags combination of following flags :
* - TCP_WRITE_FLAG_COPY (0x01) data will be copied into memory belonging to the stack
* - TCP_WRITE_FLAG_MORE (0x02) for TCP connection, PSH flag will be set on last segment sent,
* @return ERR_OK if enqueued, another err_t on error
*/
里面做了大量的事情，比如是否copy的形式进行数据传输等等，最重要的是这个过程并没有调用实际的输出函数，所以他只会对pbuf里面的缓存值一直写入知道数据超出（如果未及时发送数据），最终会导致数据无法在申请到内存，那么他到底在什么时候数据被发送的呢，从http工作原理我们可以知道http是被动形的，所以不会主动做发送，所以应该就是在rev处理完后直接调用send的

          /* If the upper layer can't receive this data, store it */
          if (err != ERR_OK) {
            pcb->refused_data = recv_data;
            LWIP_DEBUGF(TCP_INPUT_DEBUG, ("tcp_input: keep incoming packet, because pcb is \"full\"\n"));
          }
        }

        /* If a FIN segment was received, we call the callback
           function with a NULL buffer to indicate EOF. */
        if (recv_flags & TF_GOT_FIN) {
          if (pcb->refused_data != NULL) {
            /* Delay this if we have refused data. */
            pcb->refused_data->flags |= PBUF_FLAG_TCP_FIN;
          } else {
            /* correct rcv_wnd as the application won't call tcp_recved()
               for the FIN's seqno */
            if (pcb->rcv_wnd != TCP_WND) {
              pcb->rcv_wnd++;
            }
            TCP_EVENT_CLOSED(pcb, err);
            if (err == ERR_ABRT) {
              goto aborted;
            }
          }
        }

        tcp_input_pcb = NULL;
        /* Try to send something out. */
        LWIP_DEBUGF(TCP_INPUT_DEBUG, ("TCP Output data\n"));
        tcp_output(pcb);

tcp_output(pcb);就是实际的发送函数，所以会在rev函数处理完数据后才发送，这时候我们想提前做判断，看是否可以发送数据了怎么办，这里就是前面的POLL函数，POLL函数是起到轮询查看的作用，查看是否已经有数据，有的话就发生出去

static err_t
http_poll(void *arg, struct tcp_pcb *pcb)
{
  struct http_state *hs = (struct http_state *)arg;
  LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_poll: pcb=%p hs=%p pcb_state=%s\n",
    (void*)pcb, (void*)hs, tcp_debug_state_str(pcb->state)));

  if (hs == NULL) {
    err_t closed;
    /* arg is null, close. */
    LWIP_DEBUGF(HTTPD_DEBUG, ("http_poll: arg is NULL, close\n"));
    closed = http_close_conn(pcb, NULL);
    LWIP_UNUSED_ARG(closed);
#if LWIP_HTTPD_ABORT_ON_CLOSE_MEM_ERROR
    if (closed == ERR_MEM) {
       tcp_abort(pcb);
       return ERR_ABRT;
    }
#endif /* LWIP_HTTPD_ABORT_ON_CLOSE_MEM_ERROR */
    return ERR_OK;
  } else {
    hs->retries++;
    if (hs->retries == ((hs->is_websocket) ? WS_TIMEOUT : HTTPD_MAX_RETRIES)) {
      LWIP_DEBUGF(HTTPD_DEBUG, ("http_poll: too many retries, close\n"));
      http_close_conn(pcb, hs);
      return ERR_OK;
    }

    /* If this connection has a file open, try to send some more data. If
     * it has not yet received a GET request, don't do this since it will
     * cause the connection to close immediately. */
    if(hs && (hs->handle)) {
      LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_poll: try to send more data\n"));
      if(http_send(pcb, hs)) {
        /* If we wrote anything to be sent, go ahead and send it now. */
//        LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("tcp_output\n"));
        LWIP_DEBUGF(1, ("polling the data,starting send data \n"));
        tcp_output(pcb);
      }
    }
  }

  return ERR_OK;
}

这里确实是这么做的，那为什么还是会出现空间不够的情况呢，tcp_poll(pcb, http_poll, HTTPD_POLL_INTERVAL);poll的最后一个参数是用来设置轮询速度的

/** Maximum retries before the connection is aborted/closed.
 * - number of times pcb->poll is called -> default is 4*500ms = 2s;
 * - reset when pcb->sent is called
 */
#ifndef HTTPD_MAX_RETRIES
#define HTTPD_MAX_RETRIES                   4
#endif

/** The poll delay is X*500ms */
#ifndef HTTPD_POLL_INTERVAL
#define HTTPD_POLL_INTERVAL                 4
#endif

我们在看下poll函数的注释

/**
 * The poll function is called every 2nd second.
 * If there has been no data sent (which resets the retries) in 8 seconds, close.
 * If the last portion of a file has not been sent in 2 seconds, close.
 *
 * This could be increased, but we don't want to waste resources for bad connections.
 */

默认是2秒，8秒无数据就会关闭连接，所以这里连个参数都会影响到页面的显示状况，如果设置的HTTPD_MAX_RETRIES过大，会导致对于一些已经不用的连接一直在轮询而导致后面的连接无法正常使用，页面会一直在接收，如果HTTPD_POLL_INTERVAL过大，会导致发送大数据导致out of memery的情况发生。设置的过小有又会导致系统资源的浪费了，所以大家自行做个判断。

总结

虽然看似一个很小的问题，解决起来却是走过千山万水的感觉，最近有个想法，有过linux开发的同学应该都有使用dump_stack的经验，确实是调试linux内核很好的工具，所以我想做一个嵌入式平台比较通用的dump_stack，这对于代码的追踪想必是极好的，以上源码代码都在我的github上，本身问题是出在另一款芯片上的，由于还在开发且保密，在这里我就我就放esp8266的http代码，我已做了移植和修改。
have fun ！ enjoy it！
https://github.com/zwxf/ESP8266_RTOS_SDK