middle
/**
* Created by lkl on 2017/7/31.
*/
/**
* Created by lkl on 2017/6/26.
*///spark-shell --driver-class-path /home/hadoop/test/mysqljdbc.jar
import java.math.BigDecimal
import java.sql.{DriverManager, ResultSet}
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import java.text.SimpleDateFormat
import java.util.Date
object middle {
val rl= "jdbc:mysql://10.19.65.17:54321/emotion?user=emotion&password=qingxu&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false"
classOf[com.mysql.jdbc.Driver]
val conn = DriverManager.getConnection(rl)
def main(args: Array[String]) {
val conf = new SparkConf()
val sc = new SparkContext(conf)
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
val titlesplit1 = sqlContext.jdbc("jdbc:mysql://10.19.65.17:54321/emotion?user=emotion&password=qingxu", "titlesplit")
val titlesplit = titlesplit1.toDF().registerTempTable("titlesplit")
val category1 = sqlContext.jdbc("jdbc:mysql://10.19.65.17:54321/emotion?user=emotion&password=qingxu", "category")
val category = category1.toDF().registerTempTable("category")
val categorys=sqlContext.sql("select category.`id`,category.`number`,category.`category`,category.`words` from category").toDF("id","number","category","words").registerTempTable("categorys")
val layer1 = sqlContext.jdbc("jdbc:mysql://10.19.65.17:54321/emotion?user=emotion&password=qingxu", "layer")
val layer = layer1.toDF().registerTempTable("layer")
val format = new java.text.SimpleDateFormat("yyyyMMdd")
val date = format.format(new java.util.Date().getTime())
import sqlContext.implicits._
val value = sqlContext.sql("SELECT titlesplit.`innserSessionid`," +
"titlesplit.`times`,categorys.`number`," +
"categorys.`category`," +
"SUM(layer.`VALUE`) svalue," +
"COUNT(DISTINCT titlesplit.`id`) AS wordscount," +
"COUNT(DISTINCT categorys.`id`) AS categoryscount," +
"COUNT(DISTINCT categorys.`id`)/COUNT(DISTINCT titlesplit.`id`) AS rank" +
" FROM titlesplit " +
" LEFT JOIN `layer` " +
"ON titlesplit.`words`=layer.`words` " +
" LEFT JOIN categorys " +
" ON titlesplit.`words`=categorys.`words`" +
"GROUP BY titlesplit.`innserSessionid`,titlesplit.`times`," +
"categorys.`number`,categorys.`category`").toDF("innserSessionid", "times", "number","category", "svalue", "wordscount", "categoryscount", "rank")
// value.insertIntoJDBC(rl,"middles",false)
//val jo = value.toDF("innserSessionid", "times", "category", "svalue", "wordscount", "categoryscount", "rank")
val p1 = value.map(p => {
val v0 = p.getString(0)
val v1 = p.getString(1)
val v2 = p.getString(2)
val v3 = p.getString(3)
val v4 = p.getDecimal(4)
var v5 = p.getLong(5)
val v6 = p.getLong(6)
val v7 = p.getDouble(7)
(v0,v1,v2,v3,v4,v5,v6,v7)
})
p1.foreach(p => {
val v1=p._1
val v2=p._2
val v3=p._3
val v4=p._4
val v5=p._5
val v6=p._6
val v7=p._7
val v8=p._8
val format2 = new java.text.SimpleDateFormat("yyyy-MM-dd")
val dat = format2.format(new java.util.Date().getTime() - 1 * 24 * 60 * 60 * 1000).toString
if(v2==dat)
{
insert(v1,v2,v3,v4,v5,v6,v7,v8)
}
})
conn.close()
}
def insert (value0:String,value1:String,value2:String,value3:String,value4:BigDecimal,value5:Long,value6:Long,value7:Double): Unit ={
val statement = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)
// CREATE TABLE words2(innersessionId VARCHAR(100),words VARCHAR(100), VARCHAR(100),posit VARCHAR(100),va VARCHAR(100))
try {
val prep = conn.prepareStatement("INSERT INTO middle(innserSessionid,times,number,category,svalue,wordscount,categoryscount,rank) VALUES (?,?,?,?,?,?,?,?) ")
prep.setString(1,value0)
prep.setString(2,value1)
prep.setString(3,value2)
prep.setString(4,value3)
prep.setBigDecimal(5,value4)
prep.setLong(6,value5)
prep.setLong(7,value6)
prep.setDouble(8,value7)
prep.executeUpdate
} catch{
case e:Exception =>e.printStackTrace
}
finally {
}
}
}
middle的更多相关文章
- 代码的坏味道(21)——中间人(Middle Man)
坏味道--中间人(Middle Man) 特征 如果一个类的作用仅仅是指向另一个类的委托,为什么要存在呢? 问题原因 对象的基本特征之一就是封装:对外部世界隐藏其内部细节.封装往往伴随委托.但是人们可 ...
- 【BZOJ-2653】middle 可持久化线段树 + 二分
2653: middle Time Limit: 20 Sec Memory Limit: 512 MBSubmit: 1298 Solved: 734[Submit][Status][Discu ...
- Who's in the Middle[HDU1157]
Who's in the Middle Time Limit: 2000/1000 MS (Java/Others) Memory Limit: 65536/32768 K (Java/Othe ...
- [LintCode] Delete Node in the Middle of Singly Linked List 在单链表的中间删除节点
Implement an algorithm to delete a node in the middle of a singly linked list, given only access to ...
- vertical-align:middle
img style="vertical-align:middle;" src="<%=basePath%>/images/ckpy.png" > ...
- 【leetcode】Sort List (middle)
Sort a linked list in O(n log n) time using constant space complexity. 思路: 用归并排序.设输入链表为S,则先将其拆分为前半部分 ...
- 【leetcode】Reorder List (middle)
Given a singly linked list L: L0→L1→…→Ln-1→Ln,reorder it to: L0→Ln→L1→Ln-1→L2→Ln-2→… You must do thi ...
- 【leetcode】Number of Islands(middle)
Given a 2d grid map of '1's (land) and '0's (water), count the number of islands. An island is surro ...
- 运用vertical:middle 使得元素垂直居中
我要垂直居中我要垂直居中我要垂直居中我要垂直居中我要垂直居中垂直居中垂直居中垂直居中中垂直居中垂直居中 <!-- 第一步:主题元素display:inline-block;第二步:插入辅助元 ...
- Who's in the Middle 分类: POJ 2015-06-12 19:45 11人阅读 评论(0) 收藏
Who's in the Middle Time Limit: 1000MS Memory Limit: 65536K Total Submissions: 34155 Accepted: 1 ...
随机推荐
- Android padding 和margin
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" android:layout ...
- [Python] 探索性编程与idleX
1 探索式编程 前面介绍过几个Python的Interactive Shell,对于Python, Ruby这类动态语言来说,一个交互式的shell(用 emacs的术语来说,叫做REPL,也就是Re ...
- [转]关于oracle with as用法
原文地址:https://www.cnblogs.com/linjiqin/archive/2013/06/24/3152667.html with as语法–针对一个别名with tmp as (s ...
- vs2010中TargetName与链接器输出名不一致
当出现解决方案生成正确,但是无法执行.exe文件,即系统无法找到指定文件时,提示有:MSB8012: TargetName(,,,) 与 Linker 的 OutputFile 属性(,,,)不匹配. ...
- Android ——多线程处理之多线程用法大集合(转)
原文地址:http://blog.csdn.net/jie1991liu/article/details/16961701 另一篇地址:http://blog.sina.com.cn/s/blog_7 ...
- drupal8 管理入门
https://www.drupal.org/node/1896670 本节将介绍新的Drupal8用户,网站管理.它涵盖了使用管理员帐户,并提供从哪里开始的建议. 了解管理员帐户 在安装过程结束时, ...
- Redis哈希
Redis的哈希值是字符串字段和字符串值之间的映射,所以他们是表示对象的完美数据类型 在Redis中的哈希值,可存储超过400十亿键值对. 例子 redis 127.0.0.1:6379> HM ...
- Spring Cloud 5分钟搭建教程(附上一个分布式日志系统项目作为参考) - 推荐
http://blog.csdn.net/lc0817/article/details/53266212/ https://github.com/leoChaoGlut/log-sys 上面是我基于S ...
- BFS-广度优先遍历
#include <iostream> #include <queue> using namespace std; /* 5 4 0 0 1 0 0 0 0 0 0 0 1 0 ...
- Android 自定义倒计时控件CountdownTextView
效果如下: 剩余 00:59:21 package com.bihu.advertiserapp.widgets; import android.annotation.TargetApi; impor ...