Spark 读Hive并写入HBase

package com.grady

import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapred.JobConf
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession

object SparkHiveToHbase {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("sparkHiveToHbase")
    val spark: SparkSession = SparkSession.builder().config(conf).enableHiveSupport().getOrCreate()
    val sc = spark.sparkContext
    val rdd: RDD[Student] = readDataFromHive(spark)
    writeDataToHbase(rdd, sc)

    spark.stop()
  }

  def readDataFromHive(spark: SparkSession): RDD[Student] = {
    val dataFrame = spark.sql("select * from jiang.student")
    dataFrame.rdd.map(r =>
      Student(r(0).toString.toInt, r(1).toString, r(2).toString, r(3).toString.toInt, r(4).toString)
    )
  }

  def writeDataToHbase(rdd: RDD[Student], sc: SparkContext): Unit = {
    val tablename = "jiang:student"
    val hbaseConf = HBaseConfiguration.create()
    hbaseConf.set("hbase.zookeeper.quorum","10.82.232.64")
    hbaseConf.set("hbase.zookeeper.property.clientPort", "2181")
    hbaseConf.set("zookeeper.znode.parent", "/hbase")
    hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, tablename)

    val jobConf = new JobConf(hbaseConf)
    jobConf.setOutputFormat(classOf[TableOutputFormat])

    val hbaseRDD = rdd.map(student => {
      val put = new Put(Bytes.toBytes("hive:" + student.id))
      put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("name"), Bytes.toBytes(student.name))
      put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("age"), Bytes.toBytes(student.age.toString))
      (new ImmutableBytesWritable, put)
    })
    hbaseRDD.saveAsHadoopDataset(jobConf)
  }
}

case class Student(id: Int, name: String, sex: String, age: Int, department: String)

执行： spark-submit --master local[2] --num-executors 10 --class com.grady.SparkHiveToHbase /app/data/appdeploy/usehive1-1.0-SNAPSHOT.jar

日志：

hbase(main):011:0> scan 'jiang:student'
ROW                          COLUMN+CELL
 1                           column=cf:age, timestamp=2022-02-08T16:27:01.290, value=15
 1                           column=cf:name, timestamp=2022-02-08T16:27:01.290, value=jack
 2                           column=cf:age, timestamp=2022-02-08T16:27:01.290, value=16
 2                           column=cf:name, timestamp=2022-02-08T16:27:01.290, value=Lily
 3                           column=cf:age, timestamp=2022-02-08T16:27:01.290, value=16
 3                           column=cf:name, timestamp=2022-02-08T16:27:01.290, value=mike
 hive:1                      column=cf:age, timestamp=2022-02-08T17:24:24.477, value=10
 hive:1                      column=cf:name, timestamp=2022-02-08T17:24:24.477, value=xiaojiang
 hive:2                      column=cf:age, timestamp=2022-02-08T17:24:24.477, value=10
 hive:2                      column=cf:name, timestamp=2022-02-08T17:24:24.477, value=xiaochen