hive数据仓库建设

hive数据仓库建设

1、设计原生日志表

原生日志表用来存放上报的原始日志，数据经过清洗加工后会进入到各个日志表中。

1.1 创建数据库

#创建数据库
$hive>create database umeng_big11 ;

1.2 创建原生日志表

原生表使用分区表设计，分区字段为ym/d/hm，hive使用动态分区表，分区采用非严格模式，即所有分区都可以是动态分区。hive命令行终端打开显式表头设置：

#临时设置，只在当前回话有效
$hive>set hive.cli.print.header=true ;

永久配置hive-site.xml：

...
<property>
  <name>hive.cli.print.header</name>
  <value>true</value>
</property>
...

创建hive原生日志表raw_logs：

$hive>create table raw_logs
(
  servertimems	float ,
  servertimestr string ,
  clientip		string ,
  clienttimems 	bigint,
  status 		int ,
  log 			string
)
PARTITIONED BY (ym int, day int , hm int)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '#'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;

1.3 加载hdfs数据到hive原生表

$hive>use umeng_big11 ;
$hive>load data inpath '/user/centos/umeng/raw-logs/201805/24/1809' into table raw_logs partition(ym=201805 , day = 24 , hm = 1809) ;

2、自定义UDF函数完成数据清洗

2.1 介绍

将原生数据加载原生日志表后，将status码为200的记录查询出来进行清洗，将结果分别插入到5类日志子表中。供以后分析使用。

3、创建日志子表

3.1 准备sql语句

创建/home/centos/umeng/umeng_create_logs_ddl.sql文件，内容如下：

--使用指定库
use umeng_big11 ;
--startuplogs
create table if not exists startuplogs
(
  appChannel 	string ,
  appId			string ,
  appPlatform	string ,
  appVersion	string ,
  brand			string ,
  carrier 		string ,
  country 		string ,
  createdAtMs 	bigint ,
  deviceId 		string ,
  deviceStyle 	string ,
  ipAddress 	string ,
  network		string ,
  osType		string ,
  province		string ,
  screenSize	string ,
  tenantId		string
)
partitioned by (ym int ,day int , hm int)
stored as parquet ; 
--eventlogs
create table if not exists eventlogs
(
  appChannel		string ,
  appId 			string ,
  appPlatform		string ,
  appVersion		string ,
  createdAtMs		bigint ,
  deviceId			string ,
  deviceStyle		string ,
  eventDurationSecs bigint ,
  eventId 			string ,
  osType 			string ,
  tenantId 			string
)
partitioned by (ym int ,day int , hm int)
stored as parquet ; 
--errorlogs
create table if not exists errorlogs
(
  appChannel 	string ,
  appId 		string ,
  appPlatform 	string ,
  appVersion 	string ,
  createdAtMs 	bigint ,
  deviceId 		string ,
  deviceStyle 	string ,
  errorBrief 	string ,
  errorDetail 	string ,
  osType 		string ,
  tenantId 		string
)
partitioned by (ym int ,day int , hm int)
stored as parquet ; 
--usagelogs
create table if not exists usgaelogs
(
  appChannel 			string ,
  appId 				string ,
  appPlatform 			string ,
  appVersion 			string ,
  createdAtMs 			bigint ,
  deviceId 				string ,
  deviceStyle 			string ,
  osType 				string ,
  singleDownloadTraffic bigint ,
  singleUploadTraffic	bigint ,
  singleUseDurationSecs	bigint ,
  tenantId 				string
)
partitioned by (ym int ,day int , hm int)
stored as parquet ; 
--pagelogs
create table if not exists pagelogs
(
  appChannel 			string ,
  appId 				string ,
  appPlatform 			string ,
  appVersion 			string ,
  createdAtMs 			bigint ,
  deviceId 				string ,
  deviceStyle 			string ,
  nextPage 				string ,
  osType 				string ,
  pageId 				string ,
  pageViewCntInSession	int ,
  stayDurationSecs		bigint ,
  tenantId 				string ,
  visitIndex 			int
)
partitioned by (ym int ,day int , hm int)
stored as parquet ;

3.2 执行sql脚本

$hive>source /home/centos/umeng/umeng_create_logs_ddl.sql