参考:
Spring-Hadoop项目 http://blog.csdn.net/pelick/article/details/8331798
spring-hadoop之操作hbase http://www.cnblogs.com/zhangyukun/p/3685468.html
代码参考: java 获取 hbase数据 springdatahadoop — hbasetemplate http://blog.csdn.net/linlinv3/article/details/42737113
Java操作Hbase进行建表、删表以及对数据进行增删改查,条件查询 http://javacrazyer.iteye.com/blog/1186881
SHDP–Working With HBase(一)之基本介绍 http://www.cnblogs.com/MOBIN/p/5153086.html
SHDP–Working With HBase (二)之HBase JDBC驱动Phoenix与SpringJDBCTemplate的集成 http://www.cnblogs.com/MOBIN/p/5234816.html
SHDP–Working with HBase(三)之HBase+Phoenix实现分页 http://www.cnblogs.com/MOBIN/p/5235045.html
Java调用Hbase API访问接口 http://openskill.cn/article/176
Java API 与HBase交互实例, 版本比较新: http://blog.csdn.net/xw_classmate/article/details/50807372
pom.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.pandy.hadoop</groupId> <artifactId>HadoopApp</artifactId> <version>1.0-SNAPSHOT</version> <properties> <hadoop.jdbc.version>2.1.0</hadoop.jdbc.version> <hadoop.version>2.7.3</hadoop.version> </properties> <dependencies> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> <version>1.2</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.2</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> <version>4.4.5</version> </dependency> <!-- hadoop --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-jobclient</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-common</artifactId> <version>${hadoop.version}</version> </dependency> <!-- hive --> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> <version>${hadoop.jdbc.version}</version> <exclusions> <exclusion> <artifactId>pentaho-aggdesigner-algorithm</artifactId> <groupId>org.pentaho</groupId> </exclusion> </exclusions> </dependency> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-jdbc</artifactId> <version>${hadoop.jdbc.version}</version> </dependency> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-service</artifactId> <version>${hadoop.jdbc.version}</version> </dependency> <!-- hbase --> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>1.2.3</version> </dependency> <dependency> <groupId>org.springframework.data</groupId> <artifactId>spring-data-jpa</artifactId> <version>1.10.5.RELEASE</version> </dependency> <dependency> <groupId>org.springframework.data</groupId> <artifactId>spring-data-hadoop</artifactId> <version>2.4.0.RELEASE</version> </dependency> <!--<dependency> <groupId>jdk.tools</groupId> <artifactId>jdk.tools</artifactId> <version>1.7</version> <scope>system</scope> <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath> </dependency>--> <!-- Log --> <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>1.2.17</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>1.7.12</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>1.7.12</version> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> <version>1.7</version> <executions> <execution> <id>hadoop remote run</id> <phase>verify</phase> <goals> <goal>run</goal> </goals> <configuration> <target name="test"> <exec dir="${basedir}/shell" executable="bash"> <arg value="deploy.sh"></arg> </exec> </target> </configuration> </execution> </executions> </plugin> </plugins> </build> </project> |
几个配置文件
log4j.properties
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
log4j.rootLogger=DEBUG, stdout log4j.logger.org.apache=DEBUG, stdout log4j.logger.xdasLogger=DEBUG, xdas log4j.additivity.org.apache=false log4j.additivity.xdasLogger=false log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%m%n log4j.appender.xdas=org.apache.log4j.ConsoleAppender log4j.appender.xdas.layout=org.apache.log4j.PatternLayout log4j.appender.xdas.layout.ConversionPattern=%d %-5p %c - %m%n |
hbase-site.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- 这个要去Ambari的hbase config查看 --> <configuration> <property> <name>hbase.zookeeper.quorum</name> <value>h1</value> </property> <property> <name>hbase.zookeeper.property.clientPort</name> <value>2181</value> </property> <property> <name>zookeeper.znode.parent</name> <value>/hbase-unsecure</value> </property> </configuration> |
spring-hadoop.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
<?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hdp="http://www.springframework.org/schema/hadoop" xmlns:beans="http://www.springframework.org/schema/beans" xsi:schemaLocation=" http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd"> <!-- 配置zookeeper的信息,远程连接hbase时使用 --> <hdp:configuration resources="classpath:/hbase-site.xml"/> <!-- hbase --> <hdp:hbase-configuration configuration-ref="hadoopConfiguration"/> <!-- 配置HbaseTemplate --> <bean id="hbaseTemplate" class="org.springframework.data.hadoop.hbase.HbaseTemplate"> <property name="configuration" ref="hbaseConfiguration"> </property> <property name="encoding" value="UTF-8"></property> </bean> <!-- hive --> <!-- basic Hive driver bean --> <bean id="hive-driver" class="org.apache.hive.jdbc.HiveDriver"/> <!-- wrapping a basic datasource around the driver --> <bean id="hiveDataSource" class="org.springframework.jdbc.datasource.SimpleDriverDataSource"> <property name="driver" ref="hive-driver"/> <!-- 这个要去Ambari的hive config查看Database Host或者WebHCat Server host, 使用HiveServer2的主机来连接 --> <property name="url" value="jdbc:hive2://h6:10000/default"/> <property name="username" value="hive"/> <property name="password" value="ambari"/> </bean> <!-- standard JdbcTemplate declaration --> <bean id="hiveTemplate" class="org.springframework.jdbc.core.JdbcTemplate"> <property name="dataSource" ref="hiveDataSource"/> </bean> <!-- service --> <bean id="hbaseService" class="com.pandy.hadoop.hbase.HbaseService"> <property name="hbaseTemplate" ref="hbaseTemplate"/> </bean> <bean id="hiveService" class="com.pandy.hadoop.hbase.HiveService"> <property name="hiveTemplate" ref="hiveTemplate"/> </bean> </beans> |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
package com.pandy.hadoop.hbase.def;/** * Created by pandy on 16-11-17. */ import java.util.ArrayList; import java.util.List; /** * 项目名称: workspace * 功能说明: * 创建者: Pandy, * 邮箱: panyongzheng@163.com, 1453261799@qq.com * 版权: * 官网: * 创建日期: 16-11-17. * 创建时间: 下午10:00. * 修改历史: * ----------------------------------------------- */ public class HbaseFamily { private String familyName; private List<String> qualifiers; public void addQualifier(String qualifier){ if(qualifiers==null) qualifiers= new ArrayList<String>(); qualifiers.add(qualifier); } public String getFamilyName() { return familyName; } public void setFamilyName(String familyName) { this.familyName = familyName; } public List<String> getQualifiers() { return qualifiers; } public void setQualifiers(List<String> qualifiers) { this.qualifiers = qualifiers; } @Override public String toString() { return "HbaseFamily{" + "familyName='" + familyName + '\'' + ", qualifiers=" + qualifiers + '}'; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
package com.pandy.hadoop.hbase.def;/** * Created by pandy on 16-11-17. */ import java.util.ArrayList; import java.util.List; /** * 项目名称: workspace * 功能说明: * 创建者: Pandy, * 邮箱: panyongzheng@163.com, 1453261799@qq.com * 版权: * 官网: * 创建日期: 16-11-17. * 创建时间: 下午10:00. * 修改历史: * ----------------------------------------------- */ public class HbaseFamily { private String familyName; private List<String> qualifiers; public void addQualifier(String qualifier){ if(qualifiers==null) qualifiers= new ArrayList<String>(); qualifiers.add(qualifier); } public String getFamilyName() { return familyName; } public void setFamilyName(String familyName) { this.familyName = familyName; } public List<String> getQualifiers() { return qualifiers; } public void setQualifiers(List<String> qualifiers) { this.qualifiers = qualifiers; } @Override public String toString() { return "HbaseFamily{" + "familyName='" + familyName + '\'' + ", qualifiers=" + qualifiers + '}'; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
package com.pandy.hadoop.hbase.domain; public class HbaseColumnData { private String key; private String familyName; private String qualifier; private String value; private long timestamp; public String getKey() { return key; } public void setKey(String key) { this.key = key; } public String getFamilyName() { return familyName; } public void setFamilyName(String familyName) { this.familyName = familyName; } public String getQualifier() { return qualifier; } public void setQualifier(String qualifier) { this.qualifier = qualifier; } public String getValue() { return value; } public void setValue(String value) { this.value = value; } public long getTimestamp() { return timestamp; } public void setTimestamp(long timestamp) { this.timestamp = timestamp; } @Override public String toString() { return "HbaseColumn{" + "key='" + key + '\'' + ", familyName='" + familyName + '\'' + ", qualifier='" + qualifier + '\'' + ", value='" + value + '\'' + ", timestamp=" + timestamp + '}'; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
package com.pandy.hadoop.hbase.domain;/** * Created by pandy on 16-11-15. */ import java.util.List; /** * 项目名称: workspace * 功能说明: * 创建者: Pandy, * 邮箱: panyongzheng@163.com, 1453261799@qq.com * 版权: * 官网: * 创建日期: 16-11-15. * 创建时间: 上午9:22. * 修改历史: * ----------------------------------------------- */ public class HbaseRowData { private String key; private List<HbaseColumnData> columns; public HbaseColumnData getColumn(String familyName, String qualifier) { if (columns == null) return null; for (HbaseColumnData column : columns) { if (column.getFamilyName().equals(familyName) && column.getQualifier().equals(qualifier)) { return column; } } return null; } public String getKey() { return key; } public void setKey(String key) { this.key = key; } public List<HbaseColumnData> getColumns() { return columns; } public void setColumns(List<HbaseColumnData> columns) { this.columns = columns; } @Override public String toString() { return "HbaseRow{" + "key='" + key + '\'' + ", columns=" + columns + '}'; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
package com.pandy.hadoop.hbase.mapper;/** * Created by pandy on 16-11-15. */ import com.pandy.hadoop.hbase.domain.HbaseColumnData; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.springframework.data.hadoop.hbase.RowMapper; import java.util.List; /** * 项目名称: workspace * 功能说明: * 创建者: Pandy, * 邮箱: panyongzheng@163.com, 1453261799@qq.com * 版权: * 官网: * 创建日期: 16-11-15. * 创建时间: 上午9:47. * 修改历史: * ----------------------------------------------- */ public class CustomColumnMapper implements RowMapper<HbaseColumnData> { public HbaseColumnData mapRow(Result result, int rowNum) throws Exception { HbaseColumnData column = new HbaseColumnData(); List<Cell> ceList = result.listCells(); if (ceList != null && ceList.size() > 0) { for (Cell cell : ceList) { String key = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); String family = Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); String qualifier = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); long timestamp = cell.getTimestamp(); column.setKey(key); column.setValue(value); column.setFamilyName(family); column.setQualifier(qualifier); column.setTimestamp(timestamp); break; } } return column; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
package com.pandy.hadoop.hbase.mapper;/** * Created by pandy on 16-11-15. */ import com.pandy.hadoop.hbase.domain.HbaseColumnData; import com.pandy.hadoop.hbase.domain.HbaseRowData; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.springframework.data.hadoop.hbase.RowMapper; import java.util.ArrayList; import java.util.List; /** * 项目名称: workspace * 功能说明: * 创建者: Pandy, * 邮箱: panyongzheng@163.com, 1453261799@qq.com * 版权: * 官网: * 创建日期: 16-11-15. * 创建时间: 上午9:47. * 修改历史: * ----------------------------------------------- */ public class CustomRowMapper implements RowMapper<HbaseRowData> { public HbaseRowData mapRow(Result result, int rowNum) throws Exception { HbaseRowData row = new HbaseRowData(); List<HbaseColumnData> columns = new ArrayList<HbaseColumnData>(); List<Cell> ceList = result.listCells(); if (ceList != null && ceList.size() > 0) { for (Cell cell : ceList) { String key = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); String family = Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); String qualifier = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); long timestamp = cell.getTimestamp(); HbaseColumnData column = new HbaseColumnData(); column.setKey(key); column.setValue(value); column.setFamilyName(family); column.setQualifier(qualifier); column.setTimestamp(timestamp); if (row.getKey() == null) { row.setKey(key); } columns.add(column); } } row.setColumns(columns); return row; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
package com.pandy.hadoop.hbase; import org.springframework.jdbc.core.JdbcTemplate; import java.util.List; import java.util.Map; public class HiveService { private JdbcTemplate hiveTemplate; public List<Map<String, Object>> getAllUser() { List<Map<String, Object>> list = hiveTemplate.queryForList("select * from t "); for (Map<String, Object> row : list) { System.out.println("读取一行数据:" + row.get("t.id") + "\t" + row.get("t.name") + "\t" + row.get("t.idx")); } return list; } public void createUser(Integer id, String name, Integer idx) { String sql = "insert into t(id,name,idx) values(" + id + ",'" + name + "'," + idx + ")"; hiveTemplate.execute(sql); } public void createOvverUser(Integer id, String name, Integer idx) { String sql = "insert OVERWRITE t(id,name,idx) values(" + id + ",'" + name + "'," + idx + ")"; hiveTemplate.execute(sql); } public void deleteAllUser() { String sql = "truncate table t"; hiveTemplate.execute(sql); } /** * hive不支持delete * * @param id */ @Deprecated public void deleteUser(Integer id) { String sql = "delete from t where id=" + id; hiveTemplate.execute(sql); } /** * hive不支持update * * @param id * @param name * @param idx */ @Deprecated public void updateUser(Integer id, String name, Integer idx) { String sql = "update t set idx=" + idx + ", name='" + name + "' where id=" + id; hiveTemplate.execute(sql); } public JdbcTemplate getHiveTemplate() { return hiveTemplate; } public void setHiveTemplate(JdbcTemplate hiveTemplate) { this.hiveTemplate = hiveTemplate; } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 |
package com.pandy.hadoop.hbase; import com.pandy.hadoop.hbase.def.HbaseFamily; import com.pandy.hadoop.hbase.def.HbaseTable; import com.pandy.hadoop.hbase.domain.HbaseColumnData; import com.pandy.hadoop.hbase.domain.HbaseRowData; import com.pandy.hadoop.hbase.mapper.CustomColumnMapper; import com.pandy.hadoop.hbase.mapper.CustomRowMapper; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.util.Bytes; import org.springframework.data.hadoop.hbase.HbaseTemplate; import org.springframework.data.hadoop.hbase.TableCallback; import java.io.IOException; import java.util.*; public class HbaseService { private HbaseTemplate hbaseTemplate; /** * 检查表是否存在 * * @param tableName * @return */ public boolean existTable(String tableName) { HBaseAdmin hBaseAdmin = null; try { hBaseAdmin = new HBaseAdmin(hbaseTemplate.getConfiguration()); boolean isExist = hBaseAdmin.tableExists(tableName); return isExist; } catch (Exception e) { e.printStackTrace(); } finally { if (hBaseAdmin != null) { try { hBaseAdmin.close(); } catch (Exception e) { e.printStackTrace(); } } } return false; } /** * 创建表 */ public void createTable(HbaseTable table) { if (table.getFamilies() == null || table.getFamilies().size() == 0) { return; } List<HbaseFamily> families = table.getFamilies(); try { HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseTemplate.getConfiguration()); //已经存在,就增加列, 新建一个表 if (existTable(table.getTableName())) {// 如果存在要创建的表,那么先删除,再创建 System.out.println("修改表:"+table.getTableName()); Collection<HColumnDescriptor> existFamilies = hBaseAdmin.getTableDescriptor(table.getTableName().getBytes()).getFamilies(); for (HbaseFamily family : families) { Iterator<HColumnDescriptor> it = existFamilies.iterator(); boolean isExist = false; while (it.hasNext()) { HColumnDescriptor descriptor = it.next(); if (descriptor.getNameAsString().equalsIgnoreCase(family.getFamilyName())) { isExist = true; break; } } if (!isExist) { //不存在才增加 System.out.println("增加列族:" + family.getFamilyName()); hBaseAdmin.addColumn(table.getTableName(), new HColumnDescriptor(family.getFamilyName())); } } } else { System.out.println("增加表:"+table.getTableName()); HTableDescriptor tableDescriptor = new HTableDescriptor(table.getTableName()); for (HbaseFamily family : families) { System.out.println("增加列族:" + family.getFamilyName()); tableDescriptor.addFamily(new HColumnDescriptor(family.getFamilyName())); } hBaseAdmin.createTable(tableDescriptor); } } catch (Exception e) { e.printStackTrace(); } System.out.println("完成创建或者完成修改表"); } /** * 删除表 * * @param tableName */ public void dropTable(String tableName) { try { HBaseAdmin admin = new HBaseAdmin(hbaseTemplate.getConfiguration()); admin.disableTable(tableName); admin.deleteTable(tableName); } catch (MasterNotRunningException e) { e.printStackTrace(); } catch (ZooKeeperConnectionException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } /** * 写数据 * * @param tableName * @param action * @return */ public Boolean put(String tableName, final HbaseColumnData model, TableCallback<Boolean> action) { return hbaseTemplate.execute(tableName, new TableCallback<Boolean>() { public Boolean doInTable(HTableInterface table) throws Throwable { //这里得到table, 可以做非常多的故事 boolean flag = false; try { byte[] rowkey = model.getKey().getBytes(); Put put = new Put(rowkey); //put.add(Bytes.toBytes(model.getFamilyName()), Bytes.toBytes(model.getQualifier()), Bytes.toBytes(model.getValue())); put.addColumn(Bytes.toBytes(model.getFamilyName()), Bytes.toBytes(model.getQualifier()), Bytes.toBytes(model.getValue())); table.put(put); flag = true; } catch (Exception e) { e.printStackTrace(); } return flag; } }); } /** * 删除某一行 * * @param tableName * @param rowName * @return */ public boolean delete(String tableName, String rowName) { Delete delete = new Delete(rowName.getBytes()); return delete(tableName, delete); } /** * 根据需求删除数据 * * @param tableName * @param delete * @return */ public boolean delete(String tableName, final Delete delete) { return hbaseTemplate.execute(tableName, new TableCallback<Boolean>() { public Boolean doInTable(HTableInterface table) throws Throwable { //这里得到table, 可以做非常多的故事 boolean flag = false; try { table.delete(delete); flag = true; } catch (Exception e) { e.printStackTrace(); } return flag; } }); } /** * 删除列族数据 * * @param tableName * @param rowName * @param familyName */ public void delete(String tableName, String rowName, String familyName) { hbaseTemplate.delete(tableName, rowName, familyName); } /** * 删除列族里面某一列的数据 * * @param tableName * @param rowName * @param familyName * @param qualifierName */ public void delete(String tableName, String rowName, String familyName, String qualifierName) { hbaseTemplate.delete(tableName, rowName, familyName, qualifierName); } /** * 通过表名和key获取一行数据 * 包含一列以上 * * @param tableName * @param rowName * @return */ public HbaseRowData get(String tableName, String rowName) { return hbaseTemplate.get(tableName, rowName, new CustomRowMapper()); } /** * 通过表名 key 和 列族 和列 获取一个数据 * 只包含一列 * * @param tableName * @param rowName * @param familyName * @param qualifier * @return */ public HbaseColumnData get(String tableName, String rowName, String familyName, String qualifier) { return hbaseTemplate.get(tableName, rowName, familyName, qualifier, new CustomColumnMapper()); } public List<HbaseRowData> scan(String tableName) { return scan(tableName, null, null); } /** * 通过表名,开始行键和结束行键获取数据 * * @param tableName * @param startRow * @param stopRow * @return */ public List<HbaseRowData> scan(String tableName, String startRow, String stopRow) { Date d1 = new Date(); Scan scan = new Scan(); if (startRow == null) { startRow = ""; } if (stopRow == null) { stopRow = ""; } scan.setStartRow(Bytes.toBytes(startRow)); scan.setStopRow(Bytes.toBytes(stopRow)); List<HbaseRowData> list = hbaseTemplate.find(tableName, scan, new CustomRowMapper()); Date d2 = new Date(); System.out.println("查询消耗: "+((d2.getTime()-d1.getTime())*1.0/1000)+" s"); return list; } public List<HbaseRowData> scanValueByFilter(String tableName, Filter filter) throws IOException { return scanValueByMultipleFilter(tableName, filter); } public List<HbaseRowData> scanValueByMultipleFilter(String tableName, Filter... filterArr) throws IOException { Date d1 = new Date(); List<HbaseRowData> list = new ArrayList<HbaseRowData>(); HTable table = new HTable(hbaseTemplate.getConfiguration(), tableName); FilterList filters = new FilterList(); for (Filter filter : filterArr) { filters.addFilter(filter); } Scan scan = new Scan(); scan.setFilter(filters); ResultScanner resultScanner = table.getScanner(scan); for (Result rs : resultScanner) { HbaseRowData rowData = new HbaseRowData(); rowData.setKey(new String(rs.getRow())); List<HbaseColumnData> columns = new ArrayList<HbaseColumnData>(); HbaseColumnData columnData = null; Cell cell=null; columnData = new HbaseColumnData(); columnData.setKey(rowData.getKey()); columnData.setFamilyName("main"); columnData.setQualifier("name"); cell = rs.getColumnLatestCell(columnData.getFamilyName().getBytes(), columnData.getQualifier().getBytes()); columnData.setValue(getString(cell == null ? null : cell.getValue())); columnData.setTimestamp(cell == null ? 0 : cell.getTimestamp()); columns.add(columnData); columnData = new HbaseColumnData(); columnData.setKey(rowData.getKey()); columnData.setFamilyName("main"); columnData.setQualifier("title"); cell = rs.getColumnLatestCell(columnData.getFamilyName().getBytes(), columnData.getQualifier().getBytes()); columnData.setValue(getString(cell == null ? null : cell.getValue())); columnData.setTimestamp(cell == null ? 0 : cell.getTimestamp()); columns.add(columnData); columnData = new HbaseColumnData(); columnData.setKey(rowData.getKey()); columnData.setFamilyName("main"); columnData.setQualifier("birthday"); cell = rs.getColumnLatestCell(columnData.getFamilyName().getBytes(), columnData.getQualifier().getBytes()); columnData.setValue(getString(cell == null ? null : cell.getValue())); columnData.setTimestamp(cell == null ? 0 : cell.getTimestamp()); columns.add(columnData); columnData = new HbaseColumnData(); columnData.setKey(rowData.getKey()); columnData.setFamilyName("other"); columnData.setQualifier("mobile"); cell = rs.getColumnLatestCell(columnData.getFamilyName().getBytes(), columnData.getQualifier().getBytes()); columnData.setValue(getString(cell == null ? null : cell.getValue())); columnData.setTimestamp(cell == null ? 0 : cell.getTimestamp()); columns.add(columnData); columnData = new HbaseColumnData(); columnData.setKey(rowData.getKey()); columnData.setFamilyName("other"); columnData.setQualifier("email"); cell = rs.getColumnLatestCell(columnData.getFamilyName().getBytes(), columnData.getQualifier().getBytes()); columnData.setValue(getString(cell == null ? null : cell.getValue())); columnData.setTimestamp(cell == null ? 0 : cell.getTimestamp()); columns.add(columnData); rowData.setColumns(columns); list.add(rowData); System.out.println(rowData.toString()); /*System.out.println("获得到rowkey:" + new String(rs.getRow())); for (KeyValue keyValue : rs.raw()) { System.out.println("列:" + new String(keyValue.getFamily()) + "====值:" + new String(keyValue.getValue())); }*/ } Date d2 = new Date(); System.out.println("查询消耗: "+((d2.getTime()-d1.getTime())*1.0/1000)+" s"); return list; } private String getString(byte[] bytes) { if (bytes == null) return null; return new String(bytes); } public HbaseTemplate getHbaseTemplate() { return hbaseTemplate; } public void setHbaseTemplate(HbaseTemplate hbaseTemplate) { this.hbaseTemplate = hbaseTemplate; } } |
测试代码:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
package com.pandy.hadoop.hbase; import org.springframework.beans.factory.BeanFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; public class HiveTest { public static void main(String[] args) { ApplicationContext context = new ClassPathXmlApplicationContext(new String[]{"spring-hadoop.xml"}); BeanFactory factory = (BeanFactory) context; System.out.println("===========================测试HiveService"); HiveService hiveService = (HiveService) factory.getBean("hiveService"); hiveService.deleteAllUser(); System.out.println("===========================删除所有完成"); hiveService.getAllUser(); System.out.println("===========================查询完成"); hiveService.createUser(1,"pandy",1); System.out.println("===========================增加完成"); hiveService.getAllUser(); System.out.println("===========================查询完成"); } } |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 |
package com.pandy.hadoop.hbase; import com.pandy.hadoop.hbase.data.UserData; import com.pandy.hadoop.hbase.data.UserMainData; import com.pandy.hadoop.hbase.data.UserOtherData; import com.pandy.hadoop.hbase.def.HbaseFamily; import com.pandy.hadoop.hbase.def.HbaseTable; import com.pandy.hadoop.hbase.domain.HbaseColumnData; import com.pandy.hadoop.hbase.domain.HbaseRowData; import edu.jiangxin.random.RandomChinese; import edu.jiangxin.random.RandomDateTime; import edu.jiangxin.random.RandomEmail; import edu.jiangxin.random.RandomPhoneNum; import org.apache.hadoop.hbase.filter.*; import org.springframework.beans.factory.BeanFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import java.text.SimpleDateFormat; import java.util.List; public class HbaseTest { /** * 如何设计key, 才是hbase最重要的东西 * * @param flagNum * @return */ private static String generateKey(UserData user, int flagNum) { //String key = String.format("%20s", name)+"-" //System.out.println(user.toString()); String key = user.getMain().getName() + "-" + user.getMain().getTitle() + "-" + String.format("%010d", flagNum); return key; } public static void main(String[] args) throws Exception { ApplicationContext context = new ClassPathXmlApplicationContext(new String[]{"spring-hadoop-local.xml"}); BeanFactory factory = (BeanFactory) context; HbaseService hbaseService = (HbaseService) factory.getBean("hbaseService"); //RandomStringUtils.randomAlphanumeric(10), HbaseTable table = new HbaseTable(); table.setTableName("user"); HbaseFamily family = new HbaseFamily(); //主要信息 family.setFamilyName("main"); table.addFamily(family); family = new HbaseFamily(); //其他信息 family.setFamilyName("other"); table.addFamily(family); hbaseService.createTable(table); int startIdx = 0; int pad = 5000; System.out.println("===========================扫描整个表"); List<HbaseRowData> list = hbaseService.scan("user"); if (list != null) { for (HbaseRowData r : list) { System.out.println(r.toString()); } startIdx = list.size(); } System.out.println("===========================插入数据"); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); for (int i = startIdx; i <= (startIdx + pad); i++) { UserData user = new UserData(); UserMainData main = new UserMainData(); main.setName(RandomChinese.nextString(3)); main.setTitle(RandomChinese.nextString(5)); main.setBirthday(RandomDateTime.nextDate("1980-01-01", "2016-12-31")); user.setMain(main); UserOtherData other = new UserOtherData(); other.setEmail(RandomEmail.nextEmail()); other.setMobile(RandomPhoneNum.nextPhoneNum()); user.setOther(other); String key = generateKey(user, i); HbaseColumnData columnData = new HbaseColumnData(); columnData.setKey(key); columnData.setFamilyName("main"); columnData.setQualifier("name"); columnData.setValue(main.getName()); hbaseService.put("user", columnData, null); columnData = new HbaseColumnData(); columnData.setKey(key); columnData.setFamilyName("main"); columnData.setQualifier("title"); columnData.setValue(main.getTitle()); hbaseService.put("user", columnData, null); columnData = new HbaseColumnData(); columnData.setKey(key); columnData.setFamilyName("main"); columnData.setQualifier("birthday"); columnData.setValue(format.format(main.getBirthday())); hbaseService.put("user", columnData, null); columnData = new HbaseColumnData(); columnData.setKey(key); columnData.setFamilyName("other"); columnData.setQualifier("mobile"); columnData.setValue(other.getMobile()); hbaseService.put("user", columnData, null); columnData = new HbaseColumnData(); columnData.setKey(key); columnData.setFamilyName("other"); columnData.setQualifier("email"); columnData.setValue(other.getEmail()); hbaseService.put("user", columnData, null); System.out.println("KET= " + key); if (false && i > startIdx && i % 500 == 0) { try { System.out.println("===============>暂停休息一会"); Thread.sleep(10000); } catch (Exception e) { e.printStackTrace(); } } } System.out.println("===========================获得一条记录的一个列"); HbaseColumnData column = hbaseService.get("user", "愉俘埠-俱伙隘氢空-0000004018", "main", "title"); System.out.println(column.toString()); System.out.println("===========================获得一条记录"); HbaseRowData row = hbaseService.get("user", "拭邮甸-澎张吐轨蚂-0000004004"); System.out.println(row.toString()); System.out.println("===========================删除列信息"); hbaseService.delete("user", "堪伍遇-驯庚当革朵-0000004013", "main", "title"); System.out.println("===========================删除行信息"); hbaseService.delete("user", "裙毁津-璃闲戳并年-0000004006"); System.out.println("===========================使用过滤器扫描表"); hbaseService.scanValueByFilter("user", new PrefixFilter("夯".getBytes())); System.out.println("1===========================RegexStringComparator --> SingleColumnValueFilter"); hbaseService.scanValueByFilter("user", new SingleColumnValueFilter( "main".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, new RegexStringComparator("^吏"))); System.out.println("2===========================SubstringComparator --> SingleColumnValueFilter"); hbaseService.scanValueByFilter("user", new SingleColumnValueFilter( "main".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, new SubstringComparator("慈"))); System.out.println("3===========================BinaryComparator --> SingleColumnValueFilter"); hbaseService.scanValueByFilter("user", new SingleColumnValueFilter( "main".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, new BinaryComparator("藉咖串".getBytes()))); System.out.println("4===========================RegexStringComparator --> RowFilter"); hbaseService.scanValueByFilter("user", new RowFilter( CompareFilter.CompareOp.EQUAL, new RegexStringComparator("柱$"))); System.out.println("5===========================多个Filter"); hbaseService.scanValueByMultipleFilter("user", new RowFilter( CompareFilter.CompareOp.EQUAL, new RegexStringComparator("9")) , new SingleColumnValueFilter( "main".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, new RegexStringComparator("^谷"))); } } |