categorys
//spark-shell --driver-class-path /home/hadoop/test/mysqljdbc.jar
import java.sql.DriverManager
// --queue szbigdata
import org.apache.spark.{SparkConf, SparkContext}
object categorys {
val rl= "jdbc:mysql://10.19.65.17:54321/emotion?user=emotion&password=qingxu"
//jdbc:mysql://10.19.65.17:54322/emotion?user=emotion&password=qingxu
classOf[com.mysql.jdbc.Driver]
val conn = DriverManager.getConnection(rl)
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("categorys")
val sc = new SparkContext(conf)
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
val role = "jdbc:mysql://192.168.0.37:3306/emotional?user=root&password=123456&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false"
val log881101 = sc.textFile("hdfs://ns1/user/songchunlin/881101.种植业与林业.txt")
val format = new java.text.SimpleDateFormat("yyyyMMdd")
val date = format.format(new java.util.Date().getTime()).toString
val rowRDD881101= log881101.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
import sqlContext.implicits._
val slog881101 = rowRDD881101.map(line => (date, "881101", "种植业与林业", line._4)).toDF("editdate", "number", "category", "words")
val log881102 = sc.textFile("hdfs://ns1/user/songchunlin/881102.养殖业.txt")
val rowRDDlog881102=log881102.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881102=rowRDDlog881102.map(line=>(date,"881102","养殖业",line._4)).toDF("editdate", "number", "category", "words")
val log881103 = sc.textFile("hdfs://ns1/user/songchunlin/881103.农产品加工.txt")
val rowRDDlog881103=log881103.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881103=rowRDDlog881103.map(line=>(date,"881103","农产品加工",line._4)).toDF("editdate", "number", "category", "words")
val log881104 = sc.textFile("hdfs://ns1/user/songchunlin/881104.农业服务.txt")
val rowRDDlog881104=log881104.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881104=rowRDDlog881104.map(line=>(date,"881104","农业服务",line._4)).toDF("editdate", "number", "category", "words")
val log881105 = sc.textFile("hdfs://ns1/user/songchunlin/881105.煤炭开采.txt")
val rowRDDlog881105=log881105.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881105=rowRDDlog881105.map(line=>(date,"881105","煤炭开采",line._4)).toDF("editdate", "number", "category", "words")
val log881106 = sc.textFile("hdfs://ns1/user/songchunlin/881106.石油矿业开采.txt")
val rowRDDlog881106=log881106.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881106=rowRDDlog881106.map(line=>(date,"881106","石油矿业开采",line._4)).toDF("editdate", "number", "category", "words")
val log881107 = sc.textFile("hdfs://ns1/user/songchunlin/881107.采掘服务.txt")
val rowRDDlog881107=log881107.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881107=rowRDDlog881107.map(line=>(date,"881107","采掘服务",line._4)).toDF("editdate", "number", "category", "words")
val log881108 = sc.textFile("hdfs://ns1/user/songchunlin/881108.基础化学.txt")
val rowRDDlog881108=log881108.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881108=rowRDDlog881108.map(line=>(date,"881108","基础化学",line._4)).toDF("editdate", "number", "category", "words")
val log881109 = sc.textFile("hdfs://ns1/user/songchunlin/881109.化学制品.txt")
val rowRDDlog881109=log881109.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881109=rowRDDlog881109.map(line=>(date,"881109","化学制品",line._4)).toDF("editdate", "number", "category", "words")
val log881110 = sc.textFile("hdfs://ns1/user/songchunlin/881110.化工合成材料.txt")
val rowRDDlog881110=log881110.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881110=rowRDDlog881110.map(line=>(date,"881110","化工合成材料",line._4)).toDF("editdate", "number", "category", "words")
val log881111 = sc.textFile("hdfs://ns1/user/songchunlin/881111.化工新材料.txt")
val rowRDDlog881111=log881111.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881111=rowRDDlog881111.map(line=>(date,"881111","化工新材料",line._4)).toDF("editdate", "number", "category", "words")
val log881112 = sc.textFile("hdfs://ns1/user/songchunlin/881112.钢铁.txt")
val rowRDDlog881112=log881112.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881112=rowRDDlog881112.map(line=>(date,"881112","钢铁",line._4)).toDF("editdate", "number", "category", "words")
val log881113 = sc.textFile("hdfs://ns1/user/songchunlin/881113.有色冶炼加工.txt")
val rowRDDlog881113=log881113.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881113=rowRDDlog881113.map(line=>(date,"881113","有色冶炼加工",line._4)).toDF("editdate", "number", "category", "words")
val log881114 = sc.textFile("hdfs://ns1/user/songchunlin/881114.新材料.txt")
val rowRDDlog881114=log881114.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881114=rowRDDlog881114.map(line=>(date,"881114","新材料",line._4)).toDF("editdate", "number", "category", "words")
val log881115 = sc.textFile("hdfs://ns1/user/songchunlin/881115.建筑材料.txt")
val rowRDDlog881115=log881115.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881115=rowRDDlog881115.map(line=>(date,"881115","建筑材料",line._4)).toDF("editdate", "number", "category", "words")
val log881116 = sc.textFile("hdfs://ns1/user/songchunlin/881116.建筑装饰.txt")
val rowRDDlog881116=log881116.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881116=rowRDDlog881116.map(line=>(date,"881116","建筑材料",line._4)).toDF("editdate", "number", "category", "words")
val log881117 = sc.textFile("hdfs://ns1/user/songchunlin/881117.通用设备.txt")
val rowRDDlog881117=log881117.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881117=rowRDDlog881117.map(line=>(date,"881117","通用设备",line._4)).toDF("editdate", "number", "category", "words")
val log881118 = sc.textFile("hdfs://ns1/user/songchunlin/881118.专用设备.txt")
val rowRDDlog881118=log881118.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881118=rowRDDlog881118.map(line=>(date,"881118","专用设备",line._4)).toDF("editdate", "number", "category", "words")
val log881119 = sc.textFile("hdfs://ns1/user/songchunlin/881119.仪器仪表.txt")
val rowRDDlog881119=log881119.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881119=rowRDDlog881119.map(line=>(date,"881119","仪器仪表",line._4)).toDF("editdate", "number", "category", "words")
val log881120 = sc.textFile("hdfs://ns1/user/songchunlin/881120.电气设备.txt")
val rowRDDlog881120=log881120.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881120=rowRDDlog881120.map(line=>(date,"881120","电气设备",line._4)).toDF("editdate", "number", "category", "words")
val log881121 = sc.textFile("hdfs://ns1/user/songchunlin/881121.半导体及元件.txt")
val rowRDDlog881121=log881121.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881121=rowRDDlog881121.map(line=>(date,"881121","半导体及元件",line._4)).toDF("editdate", "number", "category", "words")
val log881122 = sc.textFile("hdfs://ns1/user/songchunlin/881122.光学光电子.txt")
val rowRDDlog881122=log881122.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881122=rowRDDlog881122.map(line=>(date,"881122","光学光电子",line._4)).toDF("editdate", "number", "category", "words")
val log881123 = sc.textFile("hdfs://ns1/user/songchunlin/881123.其他电子.txt")
val rowRDDlog881123=log881123.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881123=rowRDDlog881123.map(line=>(date,"881123","其他电子",line._4)).toDF("editdate", "number", "category", "words")
val log881124 = sc.textFile("hdfs://ns1/user/songchunlin/881124.电子制造.txt")
val rowRDDlog881124=log881124.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881124=rowRDDlog881124.map(line=>(date,"881124","电子制造",line._4)).toDF("editdate", "number", "category", "words")
val log881125 = sc.textFile("hdfs://ns1/user/songchunlin/881125.汽车整车.txt")
val rowRDDlog881125=log881125.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881125=rowRDDlog881125.map(line=>(date,"881125","汽车整车",line._4)).toDF("editdate", "number", "category", "words")
val log881126 = sc.textFile("hdfs://ns1/user/songchunlin/881126.汽车零部件.txt")
val rowRDDlog881126=log881126.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881126=rowRDDlog881126.map(line=>(date,"881126","汽车零部件",line._4)).toDF("editdate", "number", "category", "words")
val log881127 = sc.textFile("hdfs://ns1/user/songchunlin/881127.非汽车交运.txt")
val rowRDDlog881127=log881127.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881127=rowRDDlog881127.map(line=>(date,"881127","非汽车交运",line._4)).toDF("editdate", "number", "category", "words")
val log881128 = sc.textFile("hdfs://ns1/user/songchunlin/881128.交运设备服务.txt")
val rowRDDlog881128=log881128.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881128=rowRDDlog881128.map(line=>(date,"881128","交运设备服务",line._4)).toDF("editdate", "number", "category", "words")
val log881129 = sc.textFile("hdfs://ns1/user/songchunlin/881129.通信设备.txt")
val rowRDDlog881129=log881129.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881129=rowRDDlog881129.map(line=>(date,"881129","通信设备",line._4)).toDF("editdate", "number", "category", "words")
val log881130 = sc.textFile("hdfs://ns1/user/songchunlin/881130.计算机设备.txt")
val rowRDDlog881130=log881130.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881130=rowRDDlog881130.map(line=>(date,"881130","计算机设备",line._4)).toDF("editdate", "number", "category", "words")
val log881131 = sc.textFile("hdfs://ns1/user/songchunlin/881131.白色家电.txt")
val rowRDDlog881131 = log881131.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881131 = rowRDDlog881131.map(line => (date, "881131", "白色家电", line._4)).toDF("editdate", "number", "category", "words")
val log881132 = sc.textFile("hdfs://ns1/user/songchunlin/881132.视听器材.txt")
val rowRDDlog881132 = log881132.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881132 = rowRDDlog881132.map(line => (date, "881132", "视听器材", line._4)).toDF("editdate", "number", "category", "words")
val log881133 = sc.textFile("hdfs://ns1/user/songchunlin/881133.饮料制造.txt")
val rowRDDlog881133 = log881133.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881133 = rowRDDlog881133.map(line => (date, "881133", "饮料制造", line._4)).toDF("editdate", "number", "category", "words")
val log881134 = sc.textFile("hdfs://ns1/user/songchunlin/881134.食品加工制造.txt")
val rowRDDlog881134 = log881134.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881134 = rowRDDlog881134.map(line => (date, "881134", "食品加工制造", line._4)).toDF("editdate", "number", "category", "words")
val log881135 = sc.textFile("hdfs://ns1/user/songchunlin/881135.纺织制造.txt")
val rowRDDlog881135 = log881135.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881135 = rowRDDlog881135.map(line => (date, "881135", "纺织制造", line._4)).toDF("editdate", "number", "category", "words")
val log881136 = sc.textFile("hdfs://ns1/user/songchunlin/881136.服装家纺.txt")
val rowRDDlog881136 = log881136.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881136 = rowRDDlog881136.map(line => (date, "881136", "服装家纺", line._4)).toDF("editdate", "number", "category", "words")
val log881137 = sc.textFile("hdfs://ns1/user/songchunlin/881137.造纸.txt")
val rowRDDlog881137 = log881137.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881137 = rowRDDlog881137.map(line => (date, "881137", "造纸", line._4)).toDF("editdate", "number", "category", "words")
val log881138 = sc.textFile("hdfs://ns1/user/songchunlin/881138.包装印刷.txt")
val rowRDDlog881138 = log881138.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881138 = rowRDDlog881138.map(line => (date, "881138", "包装印刷", line._4)).toDF("editdate", "number", "category", "words")
val log881139 = sc.textFile("hdfs://ns1/user/songchunlin/881139.家用轻工.txt")
val rowRDDlog881139 = log881139.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881139 = rowRDDlog881139.map(line => (date, "881139", "家用轻工", line._4)).toDF("editdate", "number", "category", "words")
val log881140 = sc.textFile("hdfs://ns1/user/songchunlin/881140.化学制药.txt")
val rowRDDlog881140 = log881140.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881140 = rowRDDlog881140.map(line => (date, "881140", "化学制药", line._4)).toDF("editdate", "number", "category", "words")
val log881141 = sc.textFile("hdfs://ns1/user/songchunlin/881141.中药.txt")
val rowRDDlog881141 = log881141.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881141 = rowRDDlog881141.map(line => (date, "881141", "中药", line._4)).toDF("editdate", "number", "category", "words")
val log881142 = sc.textFile("hdfs://ns1/user/songchunlin/881142.生物制品.txt")
val rowRDDlog881142 = log881142.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881142 = rowRDDlog881142.map(line => (date, "881142", "生物制品", line._4)).toDF("editdate", "number", "category", "words")
val log881143 = sc.textFile("hdfs://ns1/user/songchunlin/881143.医药商业.txt")
val rowRDDlog881143 = log881143.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881143 = rowRDDlog881143.map(line => (date, "881143", "医药商业", line._4)).toDF("editdate", "number", "category", "words")
val log881144 = sc.textFile("hdfs://ns1/user/songchunlin/881144.医疗器械服务.txt")
val rowRDDlog881144 = log881144.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881144 = rowRDDlog881144.map(line => (date, "881144", "医疗器械服务", line._4)).toDF("editdate", "number", "category", "words")
val log881145 = sc.textFile("hdfs://ns1/user/songchunlin/881145.电力.txt")
val rowRDDlog881145 = log881145.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881145 = rowRDDlog881145.map(line => (date, "881145", "电力", line._4)).toDF("editdate", "number", "category", "words")
val log881146 = sc.textFile("hdfs://ns1/user/songchunlin/881146.燃气水务.txt")
val rowRDDlog881146 = log881146.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881146 = rowRDDlog881146.map(line => (date, "881146", "燃气水务", line._4)).toDF("editdate", "number", "category", "words")
val log881147 = sc.textFile("hdfs://ns1/user/songchunlin/881147.环保工程.txt")
val rowRDDlog881147 = log881147.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881147 = rowRDDlog881147.map(line => (date, "881147", "环保工程", line._4)).toDF("editdate", "number", "category", "words")
val log881148 = sc.textFile("hdfs://ns1/user/songchunlin/881148.港口航运.txt")
val rowRDDlog881148 = log881148.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881148 = rowRDDlog881148.map(line => (date, "881148", "港口航运", line._4)).toDF("editdate", "number", "category", "words")
val log881149 = sc.textFile("hdfs://ns1/user/songchunlin/881149.公路铁路运输.txt")
val rowRDDlog881149 = log881149.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881149 = rowRDDlog881149.map(line => (date, "881149", "公路铁路运输", line._4)).toDF("editdate", "number", "category", "words")
val log881150 = sc.textFile("hdfs://ns1/user/songchunlin/881150.公交.txt")
val rowRDDlog881150 = log881150.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881150 = rowRDDlog881150.map(line => (date, "881150", "公交", line._4)).toDF("editdate", "number", "category", "words")
val log881151 = sc.textFile("hdfs://ns1/user/songchunlin/881151.机场航运.txt")
val rowRDDlog881151 = log881151.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881151 = rowRDDlog881151.map(line => (date, "881151", "机场航运", line._4)).toDF("editdate", "number", "category", "words")
val log881152 = sc.textFile("hdfs://ns1/user/songchunlin/881152.物流.txt")
val rowRDDlog881152 = log881152.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881152 = rowRDDlog881152.map(line => (date, "881152", "物流", line._4)).toDF("editdate", "number", "category", "words")
val log881153 = sc.textFile("hdfs://ns1/user/songchunlin/881153.房地产开发.txt")
val rowRDDlog881153 = log881153.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881153 = rowRDDlog881153.map(line => (date, "881153", "房地产开发", line._4)).toDF("editdate", "number", "category", "words")
val log881154 = sc.textFile("hdfs://ns1/user/songchunlin/881154.园区开发.txt")
val rowRDDlog881154 = log881154.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881154 = rowRDDlog881154.map(line => (date, "881154", "园区开发", line._4)).toDF("editdate", "number", "category", "words")
val log881155 = sc.textFile("hdfs://ns1/user/songchunlin/881155.银行.txt")
val rowRDDlog881155 = log881155.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881155 = rowRDDlog881155.map(line => (date, "881155", "银行", line._4)).toDF("editdate", "number", "category", "words")
val log881156 = sc.textFile("hdfs://ns1/user/songchunlin/881156.保险及其他.txt")
val rowRDDlog881156 = log881156.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881156 = rowRDDlog881156.map(line => (date, "881156", "保险及其他", line._4)).toDF("editdate", "number", "category", "words")
val log881157 = sc.textFile("hdfs://ns1/user/songchunlin/881157.证券.txt")
val rowRDDlog881157 = log881157.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881157 = rowRDDlog881157.map(line => (date, "881157", "证券", line._4)).toDF("editdate", "number", "category", "words")
val log881158 = sc.textFile("hdfs://ns1/user/songchunlin/881158.零售.txt")
val rowRDDlog881158 = log881158.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881158 = rowRDDlog881158.map(line => (date, "881158", "零售", line._4)).toDF("editdate", "number", "category", "words")
val log881159 = sc.textFile("hdfs://ns1/user/songchunlin/881159.贸易.txt")
val rowRDDlog881159 = log881159.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881159 = rowRDDlog881159.map(line => (date, "881159", "贸易", line._4)).toDF("editdate", "number", "category", "words")
val log881160 = sc.textFile("hdfs://ns1/user/songchunlin/881160.景点及旅游.txt")
val rowRDDlog881160 = log881160.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881160 = rowRDDlog881160.map(line => (date, "881160", "景点及旅游", line._4)).toDF("editdate", "number", "category", "words")
val log881161 = sc.textFile("hdfs://ns1/user/songchunlin/881161.酒店及餐饮.txt")
val rowRDDlog881161 = log881161.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881161 = rowRDDlog881161.map(line => (date, "881161", "酒店及餐饮", line._4)).toDF("editdate", "number", "category", "words")
val log881162 = sc.textFile("hdfs://ns1/user/songchunlin/881162.通信服务.txt")
val rowRDDlog881162 = log881162.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881162 = rowRDDlog881162.map(line => (date, "881162", "通信服务", line._4)).toDF("editdate", "number", "category", "words")
val log881163 = sc.textFile("hdfs://ns1/user/songchunlin/881163.计算机应用.txt")
val rowRDDlog881163 = log881163.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
import sqlContext.implicits._
val slog881163 = rowRDDlog881163.map(line => (date, "881163", "计算机应用", line._4)).toDF("editdate", "number", "category", "words")
val s2881163 = slog881163.toDF("editdate", "number", "category", "words").registerTempTable("category")
val log881164 = sc.textFile("hdfs://ns1/user/songchunlin/881164.传媒.txt")
val rowRDDlog881164 = log881164.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881164 = rowRDDlog881164.map(line => (date, "881164", "传媒", line._4)).toDF("editdate", "number", "category", "words")
val log881165 = sc.textFile("hdfs://ns1/user/songchunlin/881165.综合.txt")
val rowRDDlog881165 = log881165.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881165 = rowRDDlog881165.map(line => (date, "881165", "综合", line._4)).toDF("editdate", "number", "category", "words")
val log881166 = sc.textFile("hdfs://ns1/user/songchunlin/881166.国防军工.txt")
val rowRDDlog881166 = log881166.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881166 = rowRDDlog881166.map(line => (date, "881166", "国防军工", line._4)).toDF("editdate", "number", "category", "words")
slog881101.insertIntoJDBC(rl,"category",false)
slog881102.insertIntoJDBC(rl,"category",false)
slog881103.insertIntoJDBC(rl,"category",false)
slog881104.insertIntoJDBC(rl,"category",false)
slog881105.insertIntoJDBC(rl,"category",false)
slog881106.insertIntoJDBC(rl,"category",false)
slog881107.insertIntoJDBC(rl,"category",false)
slog881108.insertIntoJDBC(rl,"category",false)
slog881109.insertIntoJDBC(rl,"category",false)
slog881110.insertIntoJDBC(rl,"category",false)
slog881111.insertIntoJDBC(rl,"category",false)
slog881112.insertIntoJDBC(rl,"category",false)
slog881113.insertIntoJDBC(rl,"category",false)
slog881114.insertIntoJDBC(rl,"category",false)
slog881115.insertIntoJDBC(rl,"category",false)
slog881116.insertIntoJDBC(rl,"category",false)
slog881117.insertIntoJDBC(rl,"category",false)
slog881118.insertIntoJDBC(rl,"category",false)
slog881119.insertIntoJDBC(rl,"category",false)
slog881120.insertIntoJDBC(rl,"category",false)
slog881121.insertIntoJDBC(rl,"category",false)
slog881122.insertIntoJDBC(rl,"category",false)
slog881123.insertIntoJDBC(rl,"category",false)
slog881124.insertIntoJDBC(rl,"category",false)
slog881125.insertIntoJDBC(rl,"category",false)
slog881126.insertIntoJDBC(rl,"category",false)
slog881127.insertIntoJDBC(rl,"category",false)
slog881128.insertIntoJDBC(rl,"category",false)
slog881129.insertIntoJDBC(rl,"category",false)
slog881130.insertIntoJDBC(rl,"category",false)
slog881131.insertIntoJDBC(rl,"category",false)
slog881132.insertIntoJDBC(rl,"category",false)
slog881133.insertIntoJDBC(rl,"category",false)
slog881134.insertIntoJDBC(rl,"category",false)
slog881135.insertIntoJDBC(rl,"category",false)
slog881136.insertIntoJDBC(rl,"category",false)
slog881137.insertIntoJDBC(rl,"category",false)
slog881138.insertIntoJDBC(rl,"category",false)
slog881139.insertIntoJDBC(rl,"category",false)
slog881140.insertIntoJDBC(rl,"category",false)
slog881141.insertIntoJDBC(rl,"category",false)
slog881142.insertIntoJDBC(rl,"category",false)
slog881143.insertIntoJDBC(rl,"category",false)
slog881144.insertIntoJDBC(rl,"category",false)
slog881145.insertIntoJDBC(rl,"category",false)
slog881146.insertIntoJDBC(rl,"category",false)
slog881147.insertIntoJDBC(rl,"category",false)
slog881148.insertIntoJDBC(rl,"category",false)
slog881149.insertIntoJDBC(rl,"category",false)
slog881150.insertIntoJDBC(rl,"category",false)
slog881151.insertIntoJDBC(rl,"category",false)
slog881152.insertIntoJDBC(rl,"category",false)
slog881153.insertIntoJDBC(rl,"category",false)
slog881154.insertIntoJDBC(rl,"category",false)
slog881155.insertIntoJDBC(rl,"category",false)
slog881156.insertIntoJDBC(rl,"category",false)
slog881157.insertIntoJDBC(rl,"category",false)
slog881158.insertIntoJDBC(rl,"category",false)
slog881159.insertIntoJDBC(rl,"category",false)
slog881160.insertIntoJDBC(rl,"category",false)
slog881161.insertIntoJDBC(rl,"category",false)
slog881162.insertIntoJDBC(rl,"category",false)
slog881163.insertIntoJDBC(rl,"category",false)
slog881164.insertIntoJDBC(rl,"category",false)
slog881165.insertIntoJDBC(rl,"category",false)
slog881166.insertIntoJDBC(rl,"category",false)
}
def insert(value0: String, value1: String, value2: String, value3: String): Unit = {
//val statement = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)
// CREATE TABLE words2(innersessionId VARCHAR(100),words VARCHAR(100), VARCHAR(100),posit VARCHAR(100),va VARCHAR(100))
try {
val prep = conn.prepareStatement("INSERT INTO category(editdate,number,category,words) VALUES (?,?,?,?) ")
prep.setString(1, value0)
prep.setString(2, value1)
prep.setString(3, value2)
prep.setString(4, value3)
prep.executeUpdate
} catch {
case e: Exception => e.printStackTrace
}
}
def insert2(value0: String, value1: String, value2: String, value3: String): Unit = {
try {
val prep = conn.prepareStatement("INSERT INTO category2(editdate,number,category,words) VALUES (?,?,?,?) ")
prep.setString(1, value0)
prep.setString(2, value1)
prep.setString(3, value2)
prep.setString(4, value3)
prep.executeUpdate
} catch {
case e: Exception => e.printStackTrace
}
}
}
categorys的更多相关文章
- yourphp读取分类名称{$Categorys[$r[catid]]['catname']}
页面代码: product_list.html 提供分类的id,找出分类的名称 {$Categorys[$r[catid]]['catname']}
- phpcms v9中的$CATEGORYS栏目数组
首先 如果不能用$CATEGORYS这个数组或掉不出来内容应加入 $CATEGORYS = getcache('category_content_1','commons'); 1.用途 $CATEGO ...
- categorys源码
CREATE TABLE `category` ( `id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, `editdate` char(20) NOT NUL ...
- 用MongoDB分析合肥餐饮业
看了<从数据角度解析福州美食>后难免心痒,动了要分析合肥餐饮业的念头,因此特地写了Node.js爬虫爬取了合肥的大众点评数据.分析数据库我并没有采用MySQL而是用的MongoDB,是因为 ...
- NopCommerce 在Category 显示Vendor List列表
实现效果如下: 1.在前台Web的Category Menu显示 Vendor; 2.点击Vendor 显示Vendor List列表: 主要配置步骤: 1.运行网站 Admin 后台 Categ ...
- NopCommerce 在Category 显示 Store List列表
实现效果如下: 1.在前台Web的Category Menu显示 Store; 2.点击 Store 显示 Store List列表: 3.点击 列表Store 的 Company Name 进入该S ...
- phpcms v9常用方法
1.联动菜单根据地区id显示地区名称的方法: 显示效果: 四川 >> 攀枝花 >> 仁和区 [字段名字为 area] {get_linkage($info['area'],1, ...
- MVC4做网站后台:栏目管理2、修改栏目
接上节添加栏目. 修改栏目与添加栏目非常相似,主要区别在于先向视图传递要修改栏目的model.另外在保存时比添加栏目验证要更多一些. 1.要验证父栏目不能是其本身: 2.父栏目不能是其子栏目: 3.父 ...
- MVC4做网站后台:栏目管理1、添加栏目
把栏目添加删除跟前台混在一起结构清晰,现在有了后台管理的区域就把后台管理相关的代码分开. 要实现功能: 1.添加栏目 2.删除栏目 3.修改栏目信息 -- 一.开始 1.添加 接口InterfaceC ...
随机推荐
- spring util list
spring 3.0 after <util:list/>元素 借助<list/>元素,开发者能够定义java.util.List集合.下面摘录了list.xml中的配置信息. ...
- MySQL数据库权限管理
# MySQL数据库权限管理 ### 前言------------------------------ 对不同的用户赋予不同级别的访问权限是个好习惯- 杜绝一些恶意用户 ### 参考资料------- ...
- linux find 命令查找文件和文件夹
查找目录:find /(查找范围) -name '查找关键字' -type d查找文件:find /(查找范围) -name 查找关键字 -print 详解: find命令用来在指定目录下查找文件.任 ...
- Yum Priorities
简介 Linux 发行版比较多,同时还有很多个人或组织维护了某些特定用途的安装/升级源.Yum Priorities 插件可以用来强制保护源.它通过给各个源设定不同的优先级,使得系统管理员可以将某些源 ...
- ParseUrl
#!/usr/bin/python # coding:utf-8 import re import urlparse # 解析url def ParseUrl(url): if not re.sear ...
- java中定义enum示例
/** * Enumeration for the message delivery mode. Can be persistent or * non persistent. Use the meth ...
- 一键安装mysql
#!/bin/bash # # Rrogram: # install mysql # History: # -- luoqi v0. release # email: # @qq.com #以下rpm ...
- 【Android】按钮点击事件的常用写法
学习总结: 最近学习了Android点击事件的常用写法.点击事件会触发监听对象身上的回调,常用写法有以下四种: 方法一:使用匿名内部类. public class MainActivity exten ...
- 配置Chrome Workspace功能
配置Chrome Workspace功能 Chrome Workspace功能是将在Chrome开发者工具(F12)中对文档的修改直接应用于对应文件中.由于Chrome并不知道当然文档对应用的文件为哪 ...
- RavenDb学习(四)处理文档相关性
RavenDb是文档型数据库,但是我们常常也需要定义对象之间的关系,那RavenDb当中是如何处理的呢? RavenDb提供了优雅的解决方式,使用正确的话,可以减少数据开销以及网络拥堵 Denorma ...