自动截取sql并合并，生成执行HQL

### 提取SQL语句
FILE_PATH="/data/SCRIPT"
cat tables | while read file
do
    echo "-----> ${file}"
    start=`grep -nEi "^insert|^with" ${FILE_PATH}/${file}|awk -F':' '{print $1}'|head -`
    end=$((`grep -nEi "^exitCodeCheck" ${FILE_PATH}/${file}|awk -F':' '{print $1}'` - ))
    sed -n "${start},${end}p" ${FILE_PATH}/${file}|sed 's/;$//g'
    echo ";"
done > sql_script

### 生成语法测试
FILE_PATH="/data/SCRIPT"
cat tables | while read file
do
    echo "-----> ${file}"
    start=`grep -nEi "^insert|^with" ${FILE_PATH}/${file}|awk -F':' '{print $1}'|head -`
    end=$((`grep -nEi "^exitCodeCheck" ${FILE_PATH}/${file}|awk -F':' '{print $1}'` - ))
    sed -n "${start},${end}p" ${FILE_PATH}/${file}|sed 's/;$//g'
    echo "limit 0"
    echo ";"
done > sql_script_limit

# 替换文件中的dos回车
sed -i 's#^M$#\n#g' sql_script
### 文件dos 转 unix
find . -type f|xargs -i dos2unix {}

FILE_PATH="/home/userNmae/hive_data_init/"
script="sql_script"
run_script="run_${script}"
cat /dev/null > ${run_script}
for i in {'2018-02-28','2018-04-30','2018-06-30','2018-08-31','2018-09-03'}
do
    etl_dt_start=${i}
    etl_dt_end=`date -d "1 days ${etl_dt_start}" +%Y-%m-%d`
    echo ${etl_dt_start} ${etl_dt_end}
    etl_dt=`echo ${etl_dt_start}|sed 's/-/_/g'`
    # etl_dt=`echo ${etl_dt_start_str::}`;
    echo "use cmrh_dw;"                                         > ${script}_${etl_dt}
    echo "set hive.execution.engine=spark;"                    >> ${script}_${etl_dt}
    echo "set hive.exec.dynamic.partition=true;"               >> ${script}_${etl_dt}
    echo "set hive.exec.dynamic.partition.mode=nonstrict;"     >> ${script}_${etl_dt}
    echo "---------> ${etl_dt_start} ${etl_dt_end} <---------" >> ${script}_${etl_dt}
    echo "set hivevar:IncStartAll=${etl_dt_start};"            >> ${script}_${etl_dt}
    echo "set hivevar:IncEndAll=${etl_dt_end};"                >> ${script}_${etl_dt}
    echo "source ${FILEPATH}/${script};"                       >> ${script}_${etl_dt}
    echo "hive -v -f ${script}_${etl_dt}"                      >> ${run_script}
done

### 修改版脚本
### script name: prod_init.sh
tables=$
script=sql_str_${tables}
run_script=run_${script}
file_path=/data/etlscript/DB_NAME/SCRIPT/DWD/
# 读取配置文件中的脚本名，提取相应脚本在生产环境中的SQL，合并成一个文件
cat ${tables} | while read file
do
    echo "-----> ${file}"
    start=`grep -nEi "^insert|^with" ${file_path}${file}|awk -F':' '{print $1}'|head -`
    end=$((`grep -nEi "^exitCodeCheck" ${file_path}${file}|awk -F':' '{print $1}'` - ))
    sed -n "${start},${end}p" ${file_path}${file}
    # |sed 's/;$//g'
    echo ";"
done > ${script}
 
cat /dev/null > ${run_script}
#for day in {'2018-02-28','2018-04-30','2018-06-30','2018-08-31','2018-10-14'}
# 读取日期配置文件中的日期，遍历生成批次执行脚本
cat day_file|while read day
do
    etl_dt_start=${day}
    etl_dt_end=`date -d "1 days ${etl_dt_start}" +%Y-%m-%d`
    # etl_dt_end=`date -d "1 months ${etl_dt_start}" +%Y-%m-%d`
    echo ${etl_dt_start} ${etl_dt_end}
    etl_dt_start_str=`echo ${etl_dt_start}|sed 's/-/_/g'`
    echo "set hive.exec.dynamic.partition=true;"                  > ${script}_${etl_dt_start_str}
    echo "set hive.exec.dynamic.partition.mode=nonstrict;"       >> ${script}_${etl_dt_start_str}
    echo "set hive.execution.engine=spark;"                      >> ${script}_${etl_dt_start_str}
    echo "use cmrh_dw;"                                          >> ${script}_${etl_dt_start_str}
    # etl_dt_start_str=`echo ${etl_dt_start}|sed 's/-/_/g'`
    # etl_dt_start_str=`echo ${etl_dt_start_str::}`;
    echo "---------> ${etl_dt_start} ${etl_dt_end} <---------" >> ${script}_${etl_dt_start_str}
    echo "set hivevar:IncStartAll=${etl_dt_start};"            >> ${script}_${etl_dt_start_str}
    echo "set hivevar:IncEndAll=${etl_dt_end};"                >> ${script}_${etl_dt_start_str}
    echo "source ${script}"                                    >> ${script}_${etl_dt_start_str}
    echo "hive -v -f ${script}_${etl_dt_start_str} > ${script}_${etl_dt_start_str}_log 2>&1 &" >> ${run_script}
done
EOF