linux+nginx+tomcat负载均衡，实现session同步

花了一个上午的时间研究nginx+tomcat的负载均衡测试，集群环境搭建比较顺利，但是session同步的问题折腾了几个小时才搞定，现把我的过程贴上来，以备用。
软件及环境是：
虚拟机上装centos 5.5
IP为：192.168.0.51 装上nginx和tomcat  6.0.32 命名为 Tomcat1
一台win7上装tomcat  6.0.32  IP为：192.168.0.50  命名为 Tomcat2

首先装nginx，我是参照http://blog.s135.com/nginx_php_v6/ 配了一个Nginx + PHP（FastCGI）环境，然后再加上Tomcat的转发。
我的nginx.conf 修改如下

<!--<br /> <br /> Code highlighting produced by Actipro CodeHighlighter (freeware)<br /> http://www.CodeHighlighter.com/<br /> <br /> --> #######################################################################
#
# This is the main Nginx configuration file.  
#
# More information about the configuration options is available on 
#    *  the English wiki  -  http: // wiki.nginx.org/Main
#    *  the Russian documentation  -  http: // sysoev.ru/nginx/
#
#######################################################################

# ----------------------------------------------------------------------
# Main Module  -  directives that cover basic functionality
#
#   http: // wiki.nginx.org/NginxHttpMainModule
#
# ----------------------------------------------------------------------

user              nginx;
worker_processes   8 ;

error_log   / var / log / nginx / error.log;
#error_log   / var / log / nginx / error.log  notice;
#error_log   / var / log / nginx / error.log  info;

pid         / var / run / nginx.pid;


# ----------------------------------------------------------------------
# Events Module 
#
#   http: // wiki.nginx.org/NginxHttpEventsModule
#
# ----------------------------------------------------------------------

events  {
    worker_connections  10240;
}


# ----------------------------------------------------------------------
# HTTP Core Module
#
#   http: // wiki.nginx.org/NginxHttpCoreModule 
#
# ----------------------------------------------------------------------

http  {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

server_names_hash_bucket_size  128;
client_header_buffer_size  32k;
large_client_header_buffers  4  32K;
client_max_body_size 8m;


    sendfile        on;
    tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

gzip_min_length 1k;
gzip_buffers 4  16k;
gzip_http_version 1.1;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary  on;

upstream  tserver  {
        server   192.168.0.51:8080 weight=1;
        server   192.168.0.50:8080 weight=1;
 }
    
    #
    # The default server
    #
    server {
        listen       80;
        server_name  _;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

         location ~ .*\.(php|php5)?$
        {
          root  /www/ROOT;      
          #fastcgi_pass  unix:/tmp/php-cgi.sock;
         fastcgi_pass  127.0.0.1:9000;
         fastcgi_index index.php;
         include fcgi.conf;
        }

        location ~ \.(jsp|jspx|do|htm)?$ {
         proxy_set_header  Host $host;  
         proxy_set_header  X-Real-IP  $remote_addr;
         proxy_pass http://tserver;#转向tomcat处理       
        }

        location ~ (/dwr/)? {
         proxy_set_header  Host $host;
         proxy_set_header  X-Real-IP  $remote_addr;
         proxy_pass http://tserver;#转向tomcat处理
        }


        location / {
            root   /www/ROOT;
            index  index.html index.jsp default.jsp index.do default.do;
        }

        error_page  404              /404.html;
        location = /404.html {
            root   /www/ROOT;
        }

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /www/ROOT;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }

    # Load config files from the /etc/nginx/conf.d directory
    include /etc/nginx/conf.d/**//*.conf;

}
 

下一步是配置Tomcat集群。分别打开tomcat1和tomcat2下conf中server.xml
<Server port="8005" shutdown="SHUTDOWN">
<!-- Define a non-SSL HTTP/1.1 Connector on port 8080 -->
<Connector port="8080" maxHttpHeaderSize="8192"
   maxThreads="150" minSpareThreads="25" maxSpareThreads="75"
   enableLookups="false" redirectPort="8443" acceptCount="100"
   connectionTimeout="20000" disableUploadTimeout="true" />
<!-- Define an AJP 1.3 Connector on port 8009 -->
<Connector port="8009" enableLookups="false" redirectPort="8443" protocol="AJP/1.3" />

找到<Engine name="Catalina" defaultHost="localhost">
Tomcat1下的修改为<Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat1">
Tomcat2下的修改为<Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat2">

tomcat1找到
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
修改为

<!--<br /> <br /> Code highlighting produced by Actipro CodeHighlighter (freeware)<br /> http://www.CodeHighlighter.com/<br /> <br /> --> < Cluster className = " org.apache.catalina.ha.tcp.SimpleTcpCluster "  
            channelSendOptions = " 6 " >

             < Manager className = " org.apache.catalina.ha.session.DeltaManager "
                expireSessionsOnShutdown = " false "
                notifyListenersOnReplication = " true " />
             <!--
             < Manager className = " org.apache.catalina.ha.session.BackupManager "
                expireSessionsOnShutdown = " false "
                notifyListenersOnReplication = " true "
                mapSendOptions = " 6 " />
             -->
             < Channel className = " org.apache.catalina.tribes.group.GroupChannel " >
                 < Membership className = " org.apache.catalina.tribes.membership.McastService "  
                    address = " 228.0.0.5 "  
                    bind = " 192.168.0.51 "  
                    port = " 45564 "  
                    frequency = " 500 "  
                    dropTime = " 3000 " />
                 < Receiver className = " org.apache.catalina.tribes.transport.nio.NioReceiver "  
                    address = " 192.168.0.51 "  
                    autoBind = " 100 "  
                    port = " 4001 "  
                    selectorTimeout = " 100 "  
                    maxThreads = " 6 " />
                 < Sender className = " org.apache.catalina.tribes.transport.ReplicationTransmitter " >
                     < Transport className = " org.apache.catalina.tribes.transport.nio.PooledParallelSender " />
                 </ Sender >
                 < Interceptor className = " org.apache.catalina.tribes.group.interceptors.TcpFailureDetector " />
                 < Interceptor className = " org.apache.catalina.tribes.group.interceptors.MessageDispatch15Interceptor " />
             </ Channel >
             < Valve className = " org.apache.catalina.ha.tcp.ReplicationValve "  filter = " .*\.gif;.*\.js;.*\.jpg;.*\.png;.*\.htm;.*\.html;.*\.css;.*\.txt; " />
            
             < ClusterListener className = " org.apache.catalina.ha.session.JvmRouteSessionIDBinderListener " />
             < ClusterListener className = " org.apache.catalina.ha.session.ClusterSessionListener " />
         </ Cluster >

tomcat2找到
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
修改为

还需要在程序的web.xml里面</web-app>前面加入以下这句话
<distributable/>
session同步到这里设置完毕

服务器的启动顺序如下:
tomcat1 --> |tomcat2 --> |nginx

先启动tocmat1,等tomcat1启动完全的时候再启动tomcat2，等两个tocmat全启动之后，再启动

nginx

负载的时候可以用APACHE或NGINX，如果什么都不用，两台一接防问了话，可能会出错极session不同步的问题，网上也很多人介意不要做seeeion同步，这样会降低机器的性能，有一个方法可以，就是NGINX的ip_hash,这样至少可以保证客户端去防问同一台TOCMAT，除非那台TOCMAT挂了

 

一、下载下列几个 Jar包
http://memcached-session-manager.googlecode.com/files/memcached-session-manager-1.3.0.jar
http://memcached-session-manager.googlecode.com/files/msm-javolution-serializer-jodatime-1.3.0.jar
http://memcached-session-manager.googlecode.com/files/msm-javolution-serializer-cglib-1.3.0.jar
http://spymemcached.googlecode.com/files/memcached-2.4.2.jar
http://memcached-session-manager.googlecode.com/files/javolution-5.4.3.1.jar

将以下几个Jar包Copy到 $TOMCAT_HOME/lib目下，每个Tomcat节点都需要有。

二、配置Tomcat的Server.xml配置文件，样本可以点击这里下载，主要内容如下：
http://www.javabloger.com/images/2010-05/webcluster/server.xml.txt

<Context docBase="D:\webapp" path="" reloadable="true" >
<Manager className="de.javakaffee.web.msm.MemcachedBackupSessionManager"
memcachedNodes="n1:localhost:11211"
requestUriIgnorePattern=".*\.(png|gif|jpg|css|js)$"
sessionBackupAsync="false"
sessionBackupTimeout="100"
transcoderFactoryClass="de.javakaffee.web.msm.serializer.javolution.JavolutionTranscoderFactory"
copyCollectionsForSerialization="false"
/>
</Context>

每个Tomcat的节点都需要修改。