Java程序连接Hbase集群
目录
- 一、环境介绍
- 二、配置文件
-
- 2.1 pom.xml依赖
- 2.2 application.conf文件
- 三、Java API
一、环境介绍
Java 8
SpringBoot 2.3.7.RELEASE
Hbase ： 1.2.0
zookeeper：3.4.5
Hadoop：2.6
由于依赖配置，版本冲突问题搞了好久…
二、配置文件
2.1 pom.xml依赖
<dependency>
<groupId>org.apache.hbasegroupId>
<artifactId>hbase-clientartifactId>
<version>1.2.0version>
<exclusions>
<exclusion>
<artifactId>guavaartifactId>
<groupId>com.google.guavagroupId>
exclusion>
exclusions>
dependency>
<dependency>
<groupId>com.google.guavagroupId>
<artifactId>guavaartifactId>
<version>15.0version>
dependency>
<dependency>
<groupId>com.google.protobufgroupId>
<artifactId>protobuf-javaartifactId>
<version>2.5.0version>
dependency>
<dependency>
<groupId>com.typesafegroupId>
<artifactId>configartifactId>
<version>1.3.1version>
dependency>
<dependency>
<groupId>org.apache.hadoopgroupId>
<artifactId>hadoop-commonartifactId>
<version>2.6.0version>
<exclusions>
<exclusion>
<artifactId>guavaartifactId>
<groupId>com.google.guavagroupId>
exclusion>
<exclusion>
<artifactId>httpclientartifactId>
<groupId>org.apache.httpcomponentsgroupId>
exclusion>
<exclusion>
<groupId>org.slf4jgroupId>
<artifactId>slf4j-log4j12artifactId>
exclusion>
exclusions>
dependency>
<dependency>
<groupId>org.apache.hbasegroupId>
<artifactId>hbase-serverartifactId>
<version>1.2.0version>
<exclusions>
<exclusion>
<artifactId>guavaartifactId>
<groupId>com.google.guavagroupId>
exclusion>
<exclusion>
<artifactId>hbase-clientartifactId>
<groupId>org.apache.hbasegroupId>
exclusion>
<exclusion>
<artifactId>hadoop-commonartifactId>
<groupId>org.apache.hadoopgroupId>
exclusion>
<exclusion>
<artifactId>hadoop-authartifactId>
<groupId>org.apache.hadoopgroupId>
exclusion>
exclusions>
dependency>
<dependency>
<groupId>org.apache.commonsgroupId>
<artifactId>commons-lang3artifactId>
<version>3.10version>
dependency>
2.2 application.conf文件
is.kerberos=true
krd5.conf.path="/etc/krb5.conf"
hbase.conf.path="/etc/hbase/conf/hbase-site.xml"
hbase.master.principal="hbase/[email protected]"
hbase.regionserver.principal="hbase/[email protected]"
# Connection Pool settings
db.default.poolInitialSize=10
db.default.poolMaxSize=20
db.default.connectionTimeoutMillis=10000
三、Java API
package cn.besttone.warn.business.utils;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@Component
public class HbaseClient {
private static Connection connection;
private static Configuration hbaseConfig;
private static Properties properties;
public HbaseClient() throws IOException {
connection = getConn();
}
/**
* 创建连接池并初始化环境配置
*/
public void init() throws IOException {
properties = System.getProperties();
Configuration configuration = kerberos();
//实例化HBase配置类
if (hbaseConfig == null){
hbaseConfig = HBaseConfiguration.create(configuration);
}
Config load = ConfigFactory.load();
hbaseConfig.setInt("hbase.rpc.timeout",20000);
hbaseConfig.setInt("hbase.client.operation.timeout",30000);
hbaseConfig.setInt("hbase.client.scanner.timeout.period",20000);
hbaseConfig.addResource(new Path(load.getString("hbase.conf.path"))); //hbase-site.xml 文件地址
}
public static Configuration kerberos(){
Config load = ConfigFactory.load();
Configuration configuration = new Configuration();
configuration.set("hbase.client.waitforauth.enable", "true");
boolean kerberos = Boolean.parseBoolean(load.getString("is.kerberos"));
if (kerberos){
configuration.set("hbase.security.authentication", "kerberos");
configuration.set("hadoop.security.authentication", "kerberos");
String krb5Path = load.getString("krd5.conf.path"); //krb5文件地址
System.setProperty("java.security.krb5.conf", krb5Path);
UserGroupInformation.setConfiguration(configuration);
}
return configuration;
}
public static void testParam(){
Config load = ConfigFactory.load();
System.out.println("krb5:" + load.getString("krd5.conf.path"));
System.out.println("is kerberos:" + load.getString("is.kerberos"));
System.out.println("hbase-site:" + load.getString("hbase.conf.path"));
System.out.println(StringUtils.isNotBlank(load.getString("user.keytab.path")));
}
public boolean createTable(String tableName, List<String> families){
boolean result = false;
Configuration config = kerberos();
System.out.println("准备建表, 切换Admin模式");
try {
HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseConfig);
if (hBaseAdmin.tableExists(tableName)) {// 如果存在要创建的表，那么先删除，再创建
hBaseAdmin.disableTable(tableName);
hBaseAdmin.deleteTable(tableName);
System.out.println(tableName + "已经存在, 已经删除 ");
}
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
for (String family : families){
tableDescriptor.addFamily(new HColumnDescriptor(family));
}
hBaseAdmin.createTable(tableDescriptor);
result = true;
} catch (MasterNotRunningException e) {
e.printStackTrace();
} catch (ZooKeeperConnectionException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("表 " + tableName + " 已经建立完毕");
return result;
}
public List<String> getFamilyName(String tableName) throws IOException {
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tableName));
List<String> list = new ArrayList<>();
HTableDescriptor hTableDescriptor = table.getTableDescriptor();
for(HColumnDescriptor fdescriptor : hTableDescriptor.getColumnFamilies()){
list.add(fdescriptor.getNameAsString());
}
table.close();
return list;
}
public List<String> getColumnName(String rowkey,String tableName, String family) throws IOException {
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tableName));
List<String> list = new ArrayList<>();
try{
Get get = new Get(Bytes.toBytes(rowkey));
Result result = table.get(get);
Map<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(family));
for(Map.Entry<byte[], byte[]> entry:familyMap.entrySet()){
list.add(Bytes.toString(entry.getKey()));
}
} finally {
table.close();
}
return list;
}
public List<String> getRowKey(String tableName) throws IOException {
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tableName));
List<String> list = new ArrayList<>();
try{
Scan scan = new Scan();
System.out.println("正在获取表" + tableName + "的rowkey!");
ResultScanner scanner = table.getScanner(scan);
for (Result r:scanner){
list.add(new String(r.getRow()));
}
scanner.close();
} finally {
table.close();
}
System.out.println("获取表" + tableName + "的rowkey完毕!");
return list;
}
public List<Result> getOneColumnData(String tableName, String family, String column) throws IOException {
List<Result> result = new ArrayList<>();
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tableName));
try{
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
scan.addColumn(family.getBytes(),column.getBytes());
for (Result r:scanner){
result.add(r);
}
scanner.close();
} finally {
table.close();
}
return result;
}
public List<Result> getAllData(String tableName) throws IOException {
List<Result> result = new ArrayList<>();
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tableName));
try{
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
for (Result r:scanner){
result.add(r);
}
scanner.close();
} finally {
table.close();
}
return result;
}
public Result getOneRow(String tableName,String rowkey) throws IOException {
Result r = new Result();
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tableName));
try{
Get get = new Get(rowkey.getBytes());
r = table.get(get);
} catch (IOException e) {
e.printStackTrace();
}finally {
table.close();
}
return r;
}
public void insertOneCell(String tablename, String rowKey, String family, String column, String value)
throws IOException {
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tablename));
Put put = new Put(rowKey.getBytes());
put.addColumn(family.getBytes(),column.getBytes(),value.getBytes());
try {
table.put(put);
} catch (NoSuchColumnFamilyException x){
System.out.println("\n 目标表没有" + family + "字段， 无法写入数据 \n");
} catch (IOException e) {
e.printStackTrace();
}
table.close();
}
public List<KeyValue> selectRowKeyFamilyColumn(String tablename, String rowKey, String family, String column)
throws IOException {
List<KeyValue> result = new ArrayList<>();
Connection connection = HbaseClient.connection;
if(connection.isClosed() || connection == null){
connection = this.getConn();
}
HTable table = (HTable) connection.getTable(TableName.valueOf(tablename));
Get g = new Get(rowKey.getBytes());
g.addColumn(family.getBytes(), column.getBytes());
Result rs = table.get(g);
for (KeyValue kv : rs.raw())
{
result.add(kv);
}
return result;
}
public void getNameSpace()
throws IOException {
List<KeyValue> result = new ArrayList<>();
Connection connection = HbaseClient.connection;
if(connection.isClosed()){
connection = this.getConn();
}
HBaseAdmin admin = (HBaseAdmin) connection.getAdmin();
NamespaceDescriptor[] namespaceDescriptors = admin.listNamespaceDescriptors();
for (NamespaceDescriptor namespaceDescriptor : namespaceDescriptors) {
//namespace名称
System.out.println("获取到 namespace :" + namespaceDescriptor.getName());
}
}
public Connection getConn() throws IOException {
if (connection == null ){
init();
connection = ConnectionFactory.createConnection(hbaseConfig);
System.out.println("获取 Hbase 连接成功");
}
return connection;
}
public void close() throws IOException {
if (!connection.isClosed()){
connection.close();
System.out.println("Hbase 连接关闭成功");
}
}
public static void main(String[] args) throws IOException {
testParam();
HbaseClient hbaseClient = new HbaseClient();
hbaseClient.getNameSpace();
hbaseClient.getFamilyName("stream:tableTest").
forEach(System.out::println);
}
}