Springboot+Mybatis+通用Mapper多数据源实现数据同步
有个需求需要查询远程数据库，然后将远程数据库中的数据抓取到本地，远程数据库中的数据是每天都增加的，所以就需要写个程序自动实现实时抓取。
这里我用到的框架是Springboot2.0+Mybatis+Mapper，涉及的数据库有SqlServer，Oracle，Mysql
添加依赖
org.springframework.boot
spring-boot-starter-quartz
org.quartz-scheduler
quartz
2.3.0
org.springframework.boot
spring-boot-starter-web
org.springframework.boot
spring-boot-starter-jdbc
org.mybatis.spring.boot
mybatis-spring-boot-starter
1.3.1
org.springframework.boot
spring-boot-devtools
runtime
org.projectlombok
lombok
true
org.springframework.boot
spring-boot-starter-test
test
commons-lang
commons-lang
2.6
com.microsoft.sqlserver
sqljdbc4
4.4
com.oracle
ojdbc14
10.2.0.4.0
tk.mybatis
mapper-spring-boot-starter
1.2.4
com.alibaba
druid
1.0.28
org.slf4j
slf4j-api
1.7.25
如果没有log包，可能在配置文件中的DruidDataSource会出现错误提示，但实际上不会影响到程序。
application.yml项目配置
也可以用application.properties格式配置，但是感觉yml格式看起来比较直观.
##端口
server:
port:8081
logging:
level:
cn.javabb.bootdemo.mapper: info
spring:
remote-datasource:
url: jdbc:sqlserver://66.66.66.66:1433;DatabaseName=dbName
username: sa
password: 123
#使用Druid的数据源
type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
filters: stat
maxActive: 20
initialSize: 1
maxWait: 60000
minIdle: 1
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
local-datasource:
url: jdbc:sqlserver://77.77.77.77:1433;DatabaseName=dbName
username: sa
password: 1234
#使用Druid的数据源
type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
filters: stat
maxActive: 20
initialSize: 1
maxWait: 60000
minIdle: 1
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
oracle-datasource:
url: jdbc:oracle:thin:@88.88.88.88:1521/sid
username: tiger
password: tiger
#使用Druid的数据源
type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: oracle.jdbc.driver.OracleDriver
filters: stat
maxActive: 20
initialSize: 1
maxWait: 60000
minIdle: 1
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
#Mybatis
mybatis:
configuration:
map-underscore-to-camel-case: true
mapper-locations: classpath:mapper/*.xml
typeAliasesPackage: cn.javabb.**.entity
#Mapper
mapper:
mappers:
- cn.javabb.bootdemo.base.BaseMapper
not-empty: false
identity: sqlserver
before: true其中remote-datasource,oracle-datasource和local-datasource为两个不同的数据源，mybatis配置和通用mapper配置一样的,如果不需要用到通用mapper可以去掉配置。
application启动类：
@EnableAsync
@EnableScheduling
@ServletComponentScan
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class,DataSourceTransactionManagerAutoConfiguration.class, MybatisAutoConfiguration.class})
public class BootDemoApplication {
public static void main(String[] args) {
SpringApplication.run(BootDemoApplication.class, args);
}
}注解说明：
@EnableAsync：开启异步任务支持，如果不需要可以去掉
@EnableScheduling：开启定时任务支持
@ServletComponentScan：用来扫描mapper
@SpringBootApplication:springboot注解支持
注意：如果项目依赖中有，mybatis-spring-boot-starter，会默认自动根据配置文件配置mybatis,如果刚开始项目没有配置DataSource，启动项目就会报错，在这里去掉DataSource的默认自动配置类不加载，使项目在使用的时候选择去DataSource。
数据源配置
有几个数据源配置几个DataSource
配置第一个数据源：local-datasource
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import tk.mybatis.spring.annotation.MapperScan;
import javax.sql.DataSource;
@Configuration
@MapperScan(basePackages = "cn.javabb.bootdemo.mapper.local",sqlSessionFactoryRef = "localSqlSessionFactory")
public class LocalDataSourceConfig {
@Bean(name = "localDataSource")
@ConfigurationProperties("spring.local-datasource")
public DataSource localDataSource(){
return DataSourceBuilder.create().build();
}
@Bean(name = "localSqlSessionFactory")
public SqlSessionFactory sqlSessionFactory(@Qualifier("localDataSource") DataSource dataSource) throws Exception {
SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
sessionFactoryBean.setDataSource(dataSource);
sessionFactoryBean.setMapperLocations(new PathMatchingResourcePatternResolver()
.getResources("classpath*:mapper/local/*.xml"));
return sessionFactoryBean.getObject();
}
@Bean(name = "localTransactionManager")
public DataSourceTransactionManager localTransactionManager(){
return new DataSourceTransactionManager(localDataSource());
}
}spring.local-datasource 为application.yml中配置的数据源名称
classpath*:mapper/local/*.xml 指定local数据源的mybatis的xml文件位置
原本一个数据源的时候，可以在启动程序的时候加载DataSource，但是这里是多数据源，需要为每个数据源配置DataSource，并且需要添加MapperScan注解，（因为这里是使用的通用Mapper，所以引入的class应该是mapper的，import tk.mybatis.spring.annotation.MapperScan;，如果没用就引入org的MapperScan）
basePackages这里的包路径是需要用到这个数据源的路径，这个包下所有的接口文件都会使用这个local数据源
配置第二个数据源：remote-datasource
@Configuration
@MapperScan(basePackages = "cn.javabb.bootdemo.mapper.remote",sqlSessionFactoryRef = "remoteSqlSessionFactory")
public class RemoteDataSourceConfig {
@Bean(name = "remoteDataSource")
@Primary
@ConfigurationProperties("spring.remote-datasource")
public DataSource remoteDataSource(){
return DataSourceBuilder.create().build();
}
@Bean(name = "remoteSqlSessionFactory")
@Primary
public SqlSessionFactory sqlSessionFactory(@Qualifier("remoteDataSource") DataSource dataSource) throws Exception {
SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
sessionFactoryBean.setDataSource(dataSource);
sessionFactoryBean.setMapperLocations(new PathMatchingResourcePatternResolver()
.getResources("classpath*:mapper/remote/*.xml"));
return sessionFactoryBean.getObject();
}
@Bean(name = "remoteTransactionManager")
@Primary
public DataSourceTransactionManager remoteTransactionManager(){
return new DataSourceTransactionManager(remoteDataSource());
}
}@Primary指定默认数据源。
配置第三个数据源：oracle-datasource
@Configuration
@MapperScan(basePackages = "cn.javabb.bootdemo.mapper.oracle",sqlSessionFactoryRef = "oracleSqlSessionFactory")
public class OracleDataSourceConfig {
@Bean(name = "oracleDataSource")
@ConfigurationProperties("spring.oracle-datasource")
public DataSource oracleDataSource(){
return DataSourceBuilder.create().build();
}
@Bean(name = "oracleSqlSessionFactory")
public SqlSessionFactory sqlSessionFactory(@Qualifier("oracleDataSource") DataSource dataSource) throws Exception {
SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
sessionFactoryBean.setDataSource(dataSource);
sessionFactoryBean.setMapperLocations(new PathMatchingResourcePatternResolver()
.getResources("classpath*:mapper/oracle/*.xml"));
return sessionFactoryBean.getObject();
}
@Bean(name = "oracleTransactionManager")
public DataSourceTransactionManager oracleTransactionManager(){
return new DataSourceTransactionManager(oracleDataSource());
}
}接下来所有的接口都应该按照不同的数据源放置，对应的xml也需要按照不同的数据源分开。
按照我项目的实际例子：
Mapper（使用local数据源，下面所有的方法都会在local数据源中查询）
import cn.javabb.bootdemo.entity.*;
import org.apache.ibatis.annotations.Mapper;
import org.springframework.dao.DataAccessException;
import org.springframework.stereotype.Component;
/**
* @Author QINB [email protected]
* @CreateDate 2019/1/18/018 13:08
* @Since V1.0
*/
@Component
@Mapper
public interface StLocalMapper {
int insertPptnOne(StPptn pptn);
int insertRiverOne(StRiver river);
int insertRsvrOne(StRsvr rsvr);
}StLocalMapper
insert into ST_PPTN_R_copy1(stcd,tm,drp,intv,pdr,dyp,wth,tmdown)
values
(#{item.stcd},#{item.tm},#{item.drp},#{item.intv},#{item.pdr},#{item.dyp},#{item.wth},#{item.tmdown})
insert into ST_pptn_R(stcd,tm,drp,intv,pdr,dyp,wth,tmdown)
values (#{stcd},#{tm},#{drp},#{intv},#{pdr},#{dyp},#{wth},#{tmdown})
insert into ST_river_R(STCD,TM,Z,Q,XSA,XSAVV,XSMXV,FLWCHRCD,WPTN,MSQMT,MSAMT,MSVMT,TMDOWN)
values (#{STCD},#{TM},#{Z},#{Q},#{XSA},#{XSAVV},#{XSMXV},#{FLWCHRCD},#{WPTN},#{MSQMT},#{MSAMT},#{MSVMT},#{TMDOWN})
insert into ST_rsvr_R(STCD,TM,RZ,INQ,W,BLRZ,OTQ,RWCHRCD,RWPTN,INQDR,MSQMT,TMDOWN)
values (#{STCD},#{TM},#{RZ},#{INQ},#{W},#{BLRZ},#{OTQ},#{RWCHRCD},#{RWPTN},#{INQDR},#{MSQMT},#{TMDOWN})
编写一个service调用方法就行了
@Slf4j
@Service
public class StDataSynService {
@Autowired
StLocalMapper stLocalMapper;
@Async
public void pptnSyn(){
String nowDate = DateFormatUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss");
String yesterDay = DateFormatUtils.format(DateUtils.addDays(new Date(),-day),"yyyy-MM-dd HH:mm:ss");
log.info("[{}]扫描Pptn同步数据....",nowDate);
//需要同步的数据,往前推一个小时
List remoteList = stRemoteMapper.listPptnRemote(nowDate,yesterDay);
int total = 0;
if(null!=remoteList && remoteList.size()>0){
total = remoteList.size();
log.info("获取Pptn数据：{}条。",total);
Long startTM = System.currentTimeMillis();
//执行同步
int n=0;
for(StPptn pptn:remoteList){
n += stLocalMapper.insertPptnOne(pptn);
}
if(n>0){
if(n == total){
log.info("Pptn同步成功.用时：{}秒,需要同步记录:{},成功:{}",(System.currentTimeMillis()-startTM)/1000,total,n);
}else{
log.info("Pptn同步成功.用时：{}秒,需要同步记录:{},成功:{},失败:{}",(System.currentTimeMillis()-startTM)/1000,total,n,(total-n));
}
}
}else{
log.info("Pptn无数据同步");
}
}
} @Async:表示这个方法是异步执行，可以去掉，自己去调用service这个方法就行了。
上面的例子中我并没有用到通用Mapper，其实调用通用Mapper的方法会更简单一点