基于GOLDENGATE实现数据实时分发

数据分发是中一对多的系统配置。Oracle GoldenGate支持源端数据库同多个目标数据库之间的数据同步，同步节点可以是同构数据库，也可以是异构数据库。根据不同的数据库平台，GoldenGate支持的功能会有所差异，但数据分发的总体模式如下：

通过GoldenGate实现数据分发，实现方式可以概括为如下三种方式：

• 第一种方式：每个extract进程组对应一个datapump进程组，每个datapump进程组对应一个replicat进程组
• 第二种方式：每个extract进程组对应多个datapump进程组，每个datapump进程组对应一个replicat进程组
• 第三种方式：每个extract进程组对应一个datapump进程组，每个datapump进程组对应多个replicat进程组

当然，除了如上三种方式，根据实际情况进行灵活配置，可以演变出更多的数据分发模式，下面分别针对三种实现方式搭建实验环境，过程如下：

实验环境：

   操作系统：redhat 6.5

   数据库：mysql 5.6

   源数据库节点：redhat1 192.168.18.11

   目标数据库节点：redhat2  192.168.18.12  redhat3  192.168.18.13

    

第一种方式：

redhat1节点配置脚本如下：

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
ADD EXTRACT ext1, TRANLOG, BEGIN NOW
       
       
       
       
ADD EXTTRAIL /u01/mysqlogg/dirdat/ol, EXTRACT ext1
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/ext1.prm <<EOF
       
       
       
       
EXTRACT ext1
       
       
       
       
tranlogoptions altlogdest '/var/lib/mysql/binlog/bin.index'
       
       
       
       
SOURCEDB db1@192.168.18.11:3306, USERID ogg, PASSWORD ogg
       
       
       
       
EXTTRAIL /u01/mysqlogg/dirdat/ol
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
       
       
       
       
./ggsci <<EOF
       
       
       
       
ADD EXTRACT pump1, EXTTRAILSOURCE /u01/mysqlogg/dirdat/ol, BEGIN NOW
       
       
       
       
ADD RMTTRAIL /u01/mysqlogg/dirdat/or, EXTRACT pump1
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/pump1.prm <<EOF
       
       
       
       
EXTRACT pump1
       
       
       
       
RMTHOST 192.168.18.12, MGRPORT 7809
       
       
       
       
RMTTRAIL /u01/mysqlogg/dirdat/or
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
      
      
      
      

       
       
       
       

        
        
        
        
cd /u01/mysqlogg/
        
        
        
        
./ggsci <<EOF
        
        
        
        
start mgr
        
        
        
        
ADD EXTRACT ext2, TRANLOG, BEGIN NOW
        
        
        
        
ADD EXTTRAIL /u01/mysqlogg/dirdat/sl, EXTRACT ext2
        
        
        
        
EOF
        
        
        
        
cat > /u01/mysqlogg/dirprm/ext2.prm <<EOF
        
        
        
        
EXTRACT ext2
        
        
        
        
tranlogoptions altlogdest '/var/lib/mysql/binlog/bin.index'
        
        
        
        
SOURCEDB db1@192.168.18.11:3306, USERID ogg, PASSWORD ogg
        
        
        
        
EXTTRAIL /u01/mysqlogg/dirdat/sl
        
        
        
        
TABLE db1.t*;
        
        
        
        
EOF
        
        
        
        
./ggsci <<EOF
        
        
        
        
ADD EXTRACT pump2, EXTTRAILSOURCE /u01/mysqlogg/dirdat/sl, BEGIN NOW
        
        
        
        
ADD RMTTRAIL /u01/mysqlogg/dirdat/sr, EXTRACT pump2
        
        
        
        
EOF
        
        
        
        
cat > /u01/mysqlogg/dirprm/pump2.prm <<EOF
        
        
        
        
EXTRACT pump2
        
        
        
        
RMTHOST 192.168.18.13, MGRPORT 7809
        
        
        
        
RMTTRAIL /u01/mysqlogg/dirdat/sr
        
        
        
        
TABLE db1.t*;
        
        
        
        
EOF
       
       
       
       

redhat2节点配置代码如下：

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
dblogin sourcedb ogg@192.168.18.12:3306 userid ogg password ogg
       
       
       
       
add checkpointtable ogg.ggs_checkpoint
       
       
       
       
ADD REPLICAT rep1, EXTTRAIL /u01/mysqlogg/dirdat/or, BEGIN now
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/rep1.prm <<EOF
       
       
       
       
REPLICAT rep1
       
       
       
       
ASSUMETARGETDEFS
       
       
       
       
TARGETDB db1@192.168.18.12:3306, USERID ogg, PASSWORD ogg
       
       
       
       
MAP db1.t*, TARGET db1.*;
       
       
       
       
EOF
      
      
      
      

redhat3节点配置代码如下

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
dblogin sourcedb ogg@192.168.18.13:3306 userid ogg password ogg
       
       
       
       
add checkpointtable ogg.ggs_checkpoint
       
       
       
       
ADD REPLICAT rep2, EXTTRAIL /u01/mysqlogg/dirdat/sr, BEGIN now
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/rep2.prm <<EOF
       
       
       
       
REPLICAT rep2
       
       
       
       
ASSUMETARGETDEFS
       
       
       
       
TARGETDB db1@192.168.18.13:3306, USERID ogg, PASSWORD ogg
       
       
       
       
MAP db1.t*, TARGET db1.*;
       
       
       
       
EOF
      
      
      
      

测试结果：

ogg截图：

mysql截图：

 

结论：

1：可以实现数据的1对多分发

2：多个extract进程组重复提取数据，会增加工作负载

3：各个分发进程组之间不存在重叠区域，因此当某一进程发生故障时或需要做出调整工作时，不会影响其他分发进程组

4：比较适合于分发到不同目标数据库的数据不存在交集的情况

第二种方式：

redhat1节点配置脚本如下：

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
ADD EXTRACT ext1, TRANLOG, BEGIN NOW
       
       
       
       
ADD EXTTRAIL /u01/mysqlogg/dirdat/ol, EXTRACT ext1
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/ext1.prm <<EOF
       
       
       
       
EXTRACT ext1
       
       
       
       
tranlogoptions altlogdest '/var/lib/mysql/binlog/bin.index'
       
       
       
       
SOURCEDB db1@192.168.18.11:3306, USERID ogg, PASSWORD ogg
       
       
       
       
EXTTRAIL /u01/mysqlogg/dirdat/ol
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
       
       
       
       
./ggsci <<EOF
       
       
       
       
ADD EXTRACT pump1, EXTTRAILSOURCE /u01/mysqlogg/dirdat/ol, BEGIN NOW
       
       
       
       
ADD RMTTRAIL /u01/mysqlogg/dirdat/or, EXTRACT pump1
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/pump1.prm <<EOF
       
       
       
       
EXTRACT pump1
       
       
       
       
RMTHOST 192.168.18.12, MGRPORT 7809
       
       
       
       
RMTTRAIL /u01/mysqlogg/dirdat/or
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
       
       
       
       
./ggsci <<EOF
       
       
       
       
ADD EXTRACT pump2, EXTTRAILSOURCE /u01/mysqlogg/dirdat/ol, BEGIN NOW
       
       
       
       
ADD RMTTRAIL /u01/mysqlogg/dirdat/sr, EXTRACT pump2
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/pump2.prm <<EOF
       
       
       
       
EXTRACT pump2
       
       
       
       
RMTHOST 192.168.18.13, MGRPORT 7809
       
       
       
       
RMTTRAIL /u01/mysqlogg/dirdat/sr
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
      
      
      
      

redhat2节点配置脚本：

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
dblogin sourcedb ogg@192.168.18.12:3306 userid ogg password ogg
       
       
       
       
add checkpointtable ogg.ggs_checkpoint
       
       
       
       
ADD REPLICAT rep1, EXTTRAIL /u01/mysqlogg/dirdat/or, BEGIN now
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/rep1.prm <<EOF
       
       
       
       
REPLICAT rep1
       
       
       
       
ASSUMETARGETDEFS
       
       
       
       
TARGETDB db1@192.168.18.12:3306, USERID ogg, PASSWORD ogg
       
       
       
       
MAP db1.t*, TARGET db1.*;
       
       
       
       
EOF
      
      
      
      

redhat3节点配置脚本：

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
dblogin sourcedb ogg@192.168.18.13:3306 userid ogg password ogg
       
       
       
       
add checkpointtable ogg.ggs_checkpoint
       
       
       
       
ADD REPLICAT rep2, EXTTRAIL /u01/mysqlogg/dirdat/sr, BEGIN now
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/rep2.prm <<EOF
       
       
       
       
REPLICAT rep2
       
       
       
       
ASSUMETARGETDEFS
       
       
       
       
TARGETDB db1@192.168.18.13:3306, USERID ogg, PASSWORD ogg
       
       
       
       
MAP db1.t*, TARGET db1.*;
       
       
       
       
EOF
      
      
      
      

EOF测试结果：

特点：

    1、单一extract进程组对应多个datapump进程组，可以节省本地exttrail文件占用的空间

    2、extract进程故障会影响到所有的复制进程

    3、为每个target replicat分配一个专用datapump进程组，灵活性和性能较高

第三种方式：

redhat1节点的配置代码如下

      
      
      
      

       
       
       
       
cd /u01/mysqlogg/
       
       
       
       
./ggsci <<EOF
       
       
       
       
start mgr
       
       
       
       
ADD EXTRACT ext1, TRANLOG, BEGIN NOW
       
       
       
       
ADD EXTTRAIL /u01/mysqlogg/dirdat/ol, EXTRACT ext1
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/ext1.prm <<EOF
       
       
       
       
EXTRACT ext1
       
       
       
       
tranlogoptions altlogdest '/var/lib/mysql/binlog/bin.index'
       
       
       
       
SOURCEDB db1@192.168.18.11:3306, USERID ogg, PASSWORD ogg
       
       
       
       
EXTTRAIL /u01/mysqlogg/dirdat/ol
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
       
       
       
       
./ggsci <<EOF
       
       
       
       
ADD EXTRACT pump1, EXTTRAILSOURCE /u01/mysqlogg/dirdat/ol, BEGIN NOW
       
       
       
       
EOF
       
       
       
       
cat > /u01/mysqlogg/dirprm/pump1.prm <<EOF
       
       
       
       
EXTRACT pump1
       
       
       
       
RMTHOST 192.168.18.12, MGRPORT 7809
       
       
       
       
RMTTRAIL /u01/mysqlogg/dirdat/or
       
       
       
       
TABLE db1.t*;
       
       
       
       
RMTHOST 192.168.18.13, MGRPORT 7809
       
       
       
       
RMTTRAIL /u01/mysqlogg/dirdat/sr
       
       
       
       
TABLE db1.t*;
       
       
       
       
EOF
       
       
       
       
./ggsci <<EOF
       
       
       
       
ADD RMTTRAIL /u01/mysqlogg/dirdat/or, EXTRACT pump1
       
       
       
       
ADD RMTTRAIL /u01/mysqlogg/dirdat/sr, EXTRACT pump1
       
       
       
       
EOF
      
      
      
      

redhat2节点配置脚本：

        
        
        
        

         
         
         
         
cd /u01/mysqlogg/
         
         
         
         
./ggsci <<EOF
         
         
         
         
start mgr
         
         
         
         
dblogin sourcedb ogg@192.168.18.12:3306 userid ogg password ogg
         
         
         
         
add checkpointtable ogg.ggs_checkpoint
         
         
         
         
ADD REPLICAT rep1, EXTTRAIL /u01/mysqlogg/dirdat/or, BEGIN now
         
         
         
         
EOF
         
         
         
         
cat > /u01/mysqlogg/dirprm/rep1.prm <<EOF
         
         
         
         
REPLICAT rep1
         
         
         
         
ASSUMETARGETDEFS
         
         
         
         
TARGETDB db1@192.168.18.12:3306, USERID ogg, PASSWORD ogg
         
         
         
         
MAP db1.t*, TARGET db1.*;
         
         
         
         
EOF
        
        
        
        

redhat3节点配置脚本：

         
         
         
         

          
          
          
          
cd /u01/mysqlogg/
          
          
          
          
./ggsci <<EOF
          
          
          
          
start mgr
          
          
          
          
dblogin sourcedb ogg@192.168.18.13:3306 userid ogg password ogg
          
          
          
          
add checkpointtable ogg.ggs_checkpoint
          
          
          
          
ADD REPLICAT rep2, EXTTRAIL /u01/mysqlogg/dirdat/sr, BEGIN now
          
          
          
          
EOF
          
          
          
          
cat > /u01/mysqlogg/dirprm/rep2.prm <<EOF
          
          
          
          
REPLICAT rep2
          
          
          
          
ASSUMETARGETDEFS
          
          
          
          
TARGETDB db1@192.168.18.13:3306, USERID ogg, PASSWORD ogg
          
          
          
          
MAP db1.t*, TARGET db1.*;
          
          
          
          
EOF
         
         
         
         

测试结果：

 结论：

    1、不能实现数据分发，rmttrail始终分发到同一目标数据库

    2、官方文档的部分描述与结论不一致

更多原创文章，请到www.ezbit.ren