/**
* Created by lkl on 2018/1/16.
*/
import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.tree.GradientBoostedTrees
import org.apache.spark.mllib.tree.configuration.BoostingStrategy
import org.apache.spark.mllib.tree.model.GradientBoostedTreesModel
import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.types.{DoubleType, StringType, StructField, StructType}
import org.apache.spark.{SparkConf, SparkContext}
import scala.collection.mutable.ArrayBuffer
object abregression3Model20180116 {
def main(args: Array[String]): Unit = { val sparkConf = new SparkConf().setAppName("abregression3Model20180116")
val sc = new SparkContext(sparkConf)
val hc = new HiveContext(sc)
val data = hc.sql(s"select * from lkl_card_score.fqz_score_dataset_03train").map {
row =>
val arr = new ArrayBuffer[Double]()
//剔除label、phone字段
for (i <- 3 until row.size) {
if (row.isNullAt(i)) {
arr += 0.0
}
else if (row.get(i).isInstanceOf[Int])
arr += row.getInt(i).toDouble
else if (row.get(i).isInstanceOf[Double])
arr += row.getDouble(i)
else if (row.get(i).isInstanceOf[Long])
arr += row.getLong(i).toDouble
else if (row.get(i).isInstanceOf[String])
arr += 0.0
}
LabeledPoint(row.getInt(2).toDouble, Vectors.dense(arr.toArray))
} // Split data into training (60%) and test (40%)
val Array(trainingData, testData) = data.randomSplit(Array(0.7,0.3), seed = 11L)
// 逻辑回归是迭代算法,所以缓存训练数据的RDD
trainingData.cache()
//使用SGD算法运行逻辑回归 val boostingStrategy = BoostingStrategy.defaultParams("Regression")
boostingStrategy.setNumIterations(40) // Note: Use more iterations in practice.
boostingStrategy.treeStrategy.setMaxDepth(6)
boostingStrategy.treeStrategy.setMinInstancesPerNode(50)
val model = GradientBoostedTrees.train(data, boostingStrategy)
model.save(sc, s"hdfs://ns1/user/songchunlin/model/abregression3Model20180116") sc.makeRDD(Seq(model.toDebugString)).repartition(1).saveAsTextFile(s"hdfs://ns1/user/songchunlin/model/toDebugString/abregression3Model20180116")
// 全量data数据打分,原本用testData打分
val omodel=GradientBoostedTreesModel.load(sc,s"hdfs://ns1/user/songchunlin/model/abregression3Model20180116")
val predictionAndLabels = data.map { case LabeledPoint(label, features) =>
val prediction = omodel.predict(features)
(prediction, label)
} println("testData count = " + testData.count())
println("predictionAndLabels count = " + predictionAndLabels.count())
predictionAndLabels.map(x => {"predicts: "+x._1+"--> labels:"+x._2}).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/predictionAndLabels") val metrics = new BinaryClassificationMetrics(predictionAndLabels) val precision = metrics.precisionByThreshold precision.map({case (t, p) =>
"Threshold: "+t+"Precision:"+p
}).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/precision") val recall = metrics.recallByThreshold recall.map({case (t, r) =>
"Threshold: "+t+"Recall:"+r
}).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/recall") val beta = 2
val f2Score = metrics.fMeasureByThreshold(beta) f2Score.map(x => {"Threshold: "+x._1+"--> F-score:"+x._2+"--> Beta = 2"})
.saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/f1Score") val prc = metrics.pr
prc.map(x => {"Recall: " + x._1 + "--> Precision: "+x._2 }).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/prc") // AUPRC,精度,召回曲线下的面积
val auPRC = metrics.areaUnderPR
println("Area under precision-recall curve = " +auPRC)
sc.makeRDD(Seq("Area under precision-recall curve = " +auPRC)).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/auPRC") //roc
val roc = metrics.roc
roc.map(x => {"FalsePositiveRate:" + x._1 + "--> Recall: " +x._2}).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/roc") // AUC
val auROC = metrics.areaUnderROC
sc.makeRDD(Seq("Area under ROC = " + +auROC)).saveAsTextFile(s"hdfs://ns1/user/szdsjkf/model_training/jrsc/20171218/auROC")
println("Area under ROC = " + auROC)
//val accuracy = 1.0 * predictionAndLabels.filter(x => x._1 == x._2).count() / testData.count() val dataInstance = hc.sql(s"select * from lkl_card_score.fqz_score_dataset_03train").map {
row =>
val arr = new ArrayBuffer[Double]()
//剔除label、phone字段
for (i <- 3 until row.size) {
if (row.isNullAt(i)) {
arr += 0.0
}
else if (row.get(i).isInstanceOf[Int])
arr += row.getInt(i).toDouble
else if (row.get(i).isInstanceOf[Double])
arr += row.getDouble(i)
else if (row.get(i).isInstanceOf[Long])
arr += row.getLong(i).toDouble
else if (row.get(i).isInstanceOf[String])
arr += 0.0
}
(row(0),row(1),row(2),Vectors.dense(arr.toArray))
} val preditDataGBDT = dataInstance.map { point =>
val prediction = model.predict(point._4)
//order_id,apply_time,score
(point._1,point._2,point._3,prediction)
} //rdd转dataFrame
val rowRDD = preditDataGBDT.map(row => Row(row._1.toString,row._2.toString,row._3.toString,row._4))
val schema = StructType(
List(
StructField("order_id", StringType, true),
StructField("apply_time", StringType, true),
StructField("label", StringType, true),
StructField("score", DoubleType, true)
)
)
//将RDD映射到rowRDD,schema信息应用到rowRDD上
val scoreDataFrame = hc.createDataFrame(rowRDD,schema)
scoreDataFrame.count()
scoreDataFrame.write.mode(SaveMode.Overwrite).saveAsTable("lkl_card_score.fqz_score_dataset_03train_predict") // 自己测试建模 val balance = hc.sql(s"select * from lkl_card_score.overdue_result_all_new_woe_instant_v3_02train where label='1' limit 85152 union all select * from lkl_card_score.overdue_result_all_new_woe_instant_v3_02train where label='0'").map {
row =>
val arr = new ArrayBuffer[Double]()
//剔除label、phone字段
for (i <- 3 until row.size) {
if (row.isNullAt(i)) {
arr += 0.0
}
else if (row.get(i).isInstanceOf[Int])
arr += row.getInt(i).toDouble
else if (row.get(i).isInstanceOf[Double])
arr += row.getDouble(i)
else if (row.get(i).isInstanceOf[Long])
arr += row.getLong(i).toDouble
else if (row.get(i).isInstanceOf[String])
arr += 0.0
}
LabeledPoint(row.getInt(2).toDouble, Vectors.dense(arr.toArray))
} // 逻辑回归是迭代算法,所以缓存训练数据的RDD
balance.cache()
val boostingStrategy1 = BoostingStrategy.defaultParams("Regression")
boostingStrategy1.setNumIterations(40) // Note: Use more iterations in practice.
boostingStrategy1.treeStrategy.setMaxDepth(6)
boostingStrategy1.treeStrategy.setMinInstancesPerNode(50) val model2 = GradientBoostedTrees.train(balance, boostingStrategy1) val predictionAndLabels2 = data.map { case LabeledPoint(label, features) =>
val prediction = model2.predict(features)
(prediction, label)
}
val metrics2 = new BinaryClassificationMetrics(predictionAndLabels2)
// AUPRC,精度,召回曲线下的面积
val auPRC1 = metrics2.areaUnderPR val preditDataGBDT1 = dataInstance.map { point =>
val prediction2 = model2.predict(point._4)
//order_id,apply_time,score
(point._1,point._2,point._3,prediction2)
}
//rdd转dataFrame
val rowRDD2 = preditDataGBDT1.map(row => Row(row._1.toString,row._2.toString,row._3.toString,row._4))
val schema2 = StructType(
List(
StructField("order_id", StringType, true),
StructField("apply_time", StringType, true),
StructField("label", StringType, true),
StructField("score", DoubleType, true)
)
) val scoreDataFrame2 = hc.createDataFrame(rowRDD2,schema2)
scoreDataFrame2.count()
scoreDataFrame2.write.mode(SaveMode.Overwrite).saveAsTable("lkl_card_score.fqz_score_dataset_02val_170506_predict") }
}

lakala反欺诈建模实际应用代码GBDT监督学习的更多相关文章

  1. AI反欺诈:千亿的蓝海,烫手的山芋|甲子光年

    不久前,一家业界领先的机器学习公司告诉「甲子光年」:常有客户带着迫切的反欺诈需求主动找来,但是,我们不敢接. 难点何在? 作者|晕倒羊 编辑|甲小姐 设计|孙佳栋 生死欺诈 企业越急速发展,越容易产生 ...

  2. 反欺诈(Fraud Detection)中所用到的机器学习模型

    反欺诈应用的机器模型算法,多为二分类算法. 1.gbdt梯度提升决策树(Gradient Boosting Decision Tree,GBDT)算法,该算法的性能高,且在各类数据挖掘中应用广泛,表现 ...

  3. 基于Vue2和Node.js的反欺诈系统设计与实现

    最近包工头喊农民工小郑搬砖,小郑搬完砖后沉思片刻,决定写篇小作文分享下,作为一个初学者的全栈项目,去学习它的搭建,到落地,再到部署维护,是非常好的. ​ ------题记 写在前面 通过本文的学习,你 ...

  4. 反编译工具 使用.NET JustDecompile来反编译你的程序代码

    原文地址:http://www.it165.net/pro/html/201310/7383.html 前言 在项目的进行中有时会碰到需要去了解由第三方所开发的程序代码或者因为年久已经遗失原始码的程序 ...

  5. python金融反欺诈-项目实战

    python信用评分卡(附代码,博主录制) https://study.163.com/course/introduction.htm?courseId=1005214003&utm_camp ...

  6. Android Studio 动态调试 apk 反编译出的 smali 代码

    在信安大赛的准备过程中,主要通过 Android Studio 动态调试 apk 反编译出来的 smali 代码的方式来对我们分析的执行流程进行验证.该技巧的主要流程在此记录.以下过程使用 Andro ...

  7. 使用.NET JustDecompile来反编译你的程序代码

    前言 在项目的进行中有时会碰到需要去了解由第三方所开发的程序代码或者因为年久已经遗失原始码的程序,由于因为是别人写的所以我们并没有原始码可以直接阅读,碰到这种情况我们就需要去反编译这些程序及 DLL ...

  8. OAF_开发系列28_实现OAF中反编译获取class包代码JD Compiler(案例)

    20150730 Created By BaoXinjian

  9. 使用VBA进行JS加密的反混淆,还原JS代码。

    本文地址:http://www.cnblogs.com/Charltsing/p/JSEval.html 联系QQ:564955427 类似下面的代码是登陆 全国企业信用信息公示系统(安徽)(网址:h ...

随机推荐

  1. 线程安全的无锁RingBuffer的实现【一个读线程,一个写线程】

    在程序设计中,我们有时会遇到这样的情况,一个线程将数据写到一个buffer中,另外一个线程从中读数据.所以这里就有多线程竞争的问题.通常的解决办法是对竞争资源加锁.但是,一般加锁的损耗较高.其实,对于 ...

  2. 4. Stacked AutoEncoder(堆栈自动编码器)

    1. AutoEncoder介绍 2. Applications of AutoEncoder in NLP 3. Recursive Autoencoder(递归自动编码器) 4. Stacked ...

  3. xss绕过htmlspecialchars实体编码的姿势

    倘若是在script.input标签当中,即可突破.Payload ' oninput=alert`1` //      当要在input中输入内容时触发事件' oninput=alert`1` ' ...

  4. idea中git git pull push需要反复输入密码

    在使用idea开发的过程中,在终端terminal中git pull和git push时遇到一个问题,一个是 每次提交都需要输入用户名和密码,,从网上找了下解决方案,记录一下. 解决: 打开git终端 ...

  5. 聊下图片滤镜,手机上的,lookup table(颜色查找表

    今天这里要介绍的是lookup table(颜色查找表),简而言之就是通过将每一个原始的颜色进行转换之后成为一个新的颜色. 打一个比方,比如原始颜色是红色(r:255,g:0,b:0),进行转换后变为 ...

  6. centos7安装rabbitmq3.7

    centos7安装rabbitmq3.7安装erlang # vim /etc/yum.repos.d/rabbitmq-erlang.repo [rabbitmq-erlang] name=rabb ...

  7. React Native安卓项目打包发布APK步骤

    1.产生签名的key 该过程会用到keytool,开发过安卓的都应该接触过该东西.详细请见密钥和证书管理工具.在项目的主目录(不是android文件夹)中执行: --生成签名key,注意记下你的密钥和 ...

  8. 转:Java中String与byte[]的转换

    原文地址:http://blog.csdn.net/llwan/article/details/7567906 String s = "fs123fdsa";//String变量 ...

  9. Policy Gradient

    Policy Gradient是区别于Q-Learning为代表的value based的方法.policy gradient又可以叫reinforce算法(Williams, 1992). 如今的A ...

  10. 如何配置propagation

    配置spring事务代理时的事务属性. <prop key=“get*”>PROPAGATION_REQUIRED,readOnly</prop> 表示类方法名称是以get开头 ...