Spark 决策树--分类模型

package Spark_MLlib

import org.apache.spark.ml.Pipeline

import org.apache.spark.ml.classification.{DecisionTreeClassificationModel, DecisionTreeClassifier}

import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator

import org.apache.spark.ml.feature.{IndexToString, StringIndexer, VectorIndexer}

import org.apache.spark.ml.linalg.{Vector, Vectors}

import org.apache.spark.mllib.tree.DecisionTree

import org.apache.spark.sql.SparkSession

/**

  * Created by soyo on 17-11-5.

  */

case class data_schemas(features:Vector,label:String)

object 决策树 {

   val spark=SparkSession.builder().master("local").appName("决策树").getOrCreate()

  import spark.implicits._

  def main(args: Array[String]): Unit = {

    val source_DF=spark.sparkContext.textFile("file:///home/soyo/桌面/spark编程测试数据/soyo2.txt")

                    .map(_.split(",")).map(x=>data_schemas(Vectors.dense(x().toDouble,x().toDouble,x().toDouble,x().toDouble),x())).toDF()

        source_DF.createOrReplaceTempView("decisonTree")

    val DF=spark.sql("select * from decisonTree")

        DF.show()

    //分别获取标签列和特征列,进行索引和重命名(索引的目的是将字符串label数值化方便机器学习算法学习)

    val lableIndexer=new StringIndexer().setInputCol("label").setOutputCol("indexedLabel").fit(DF)

    val featureIndexer= new VectorIndexer().setInputCol("features").setOutputCol("indexedFeatures").setMaxCategories().fit(DF)

    val labelConverter= new IndexToString().setInputCol("prediction").setOutputCol("predictedLabel").setLabels(lableIndexer.labels)

    // 训练数据和测试数据

    val Array(trainData,testData)=DF.randomSplit(Array(0.7,0.3))

    val decisionTreeClassifier=new DecisionTreeClassifier().setLabelCol("indexedLabel").setFeaturesCol("indexedFeatures")

    //构建机器学习工作流

    val dt_pipeline=new Pipeline().setStages(Array(lableIndexer,featureIndexer,decisionTreeClassifier,labelConverter))

    val dt_model=dt_pipeline.fit(trainData)

    //进行预测

    val dtprediction=dt_model.transform(testData)

    dtprediction.show()

    //评估决策树模型

    val evaluatorClassifier=new MulticlassClassificationEvaluator().setLabelCol("indexedLabel").setPredictionCol("prediction").setMetricName("accuracy")

    val accuracy=evaluatorClassifier.evaluate(dtprediction)

    println("准确率为: "+accuracy)

    val error=-accuracy

    println("错误率为: "+error)

    val treeModelClassifier=dt_model.stages().asInstanceOf[DecisionTreeClassificationModel]

    val schema_DecisionTree=treeModelClassifier.toDebugString

    println("决策树的模型结构为: "+schema_DecisionTree)

  }

}

结果为：

+-----------------+------+
| features| label|
+-----------------+------+
|[5.1,3.5,1.4,0.2]|hadoop|
|[4.9,3.0,1.4,0.2]|hadoop|
|[4.7,3.2,1.3,0.2]|hadoop|
|[4.6,3.1,1.5,0.2]|hadoop|
|[5.0,3.6,1.4,0.2]|hadoop|
|[5.4,3.9,1.7,0.4]|hadoop|
|[4.6,3.4,1.4,0.3]|hadoop|
|[5.0,3.4,1.5,0.2]|hadoop|
|[4.4,2.9,1.4,0.2]|hadoop|
|[4.9,3.1,1.5,0.1]|hadoop|
|[5.4,3.7,1.5,0.2]|hadoop|
|[4.8,3.4,1.6,0.2]|hadoop|
|[4.8,3.0,1.4,0.1]|hadoop|
|[4.3,3.0,1.1,0.1]|hadoop|
|[5.8,4.0,1.2,0.2]|hadoop|
|[5.7,4.4,1.5,0.4]|hadoop|
|[5.4,3.9,1.3,0.4]|hadoop|
|[5.1,3.5,1.4,0.3]|hadoop|
|[5.7,3.8,1.7,0.3]|hadoop|
|[5.1,3.8,1.5,0.3]|hadoop|
+-----------------+------+
only showing top 20 rows

+-----------------+------+------------+-----------------+--------------+-------------+----------+--------------+
|         features| label|indexedLabel| indexedFeatures| rawPrediction| probability|prediction|predictedLabel|
+-----------------+------+------------+-----------------+--------------+-------------+----------+--------------+
|[4.4,3.0,1.3,0.2]|hadoop|         1.0|[4.4,3.0,1.3,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[4.6,3.4,1.4,0.3]|hadoop|         1.0|[4.6,3.4,1.4,0.3]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[4.6,3.6,1.0,0.2]|hadoop|         1.0|[4.6,3.6,1.0,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[4.9,2.4,3.3,1.0]| spark|         0.0|[4.9,2.4,3.3,1.0]| [0.0,0.0,1.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[5.0,2.0,3.5,1.0]| spark|         0.0|[5.0,2.0,3.5,1.0]| [1.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.0,2.3,3.3,1.0]| spark|         0.0|[5.0,2.3,3.3,1.0]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.0,3.2,1.2,0.2]|hadoop|         1.0|[5.0,3.2,1.2,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.0,3.3,1.4,0.2]|hadoop|         1.0|[5.0,3.3,1.4,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.0,3.4,1.6,0.4]|hadoop|         1.0|[5.0,3.4,1.6,0.4]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.0,3.6,1.4,0.2]|hadoop|         1.0|[5.0,3.6,1.4,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.1,3.5,1.4,0.2]|hadoop|         1.0|[5.1,3.5,1.4,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.1,3.7,1.5,0.4]|hadoop|         1.0|[5.1,3.7,1.5,0.4]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.2,3.4,1.4,0.2]|hadoop|         1.0|[5.2,3.4,1.4,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.2,4.1,1.5,0.1]|hadoop|         1.0|[5.2,4.1,1.5,0.1]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.4,3.0,4.5,1.5]| spark|         0.0|[5.4,3.0,4.5,1.5]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.4,3.9,1.7,0.4]|hadoop|         1.0|[5.4,3.9,1.7,0.4]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.5,2.4,3.7,1.0]| spark|         0.0|[5.5,2.4,3.7,1.0]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.5,2.4,3.8,1.1]| spark|         0.0|[5.5,2.4,3.8,1.1]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.5,2.5,4.0,1.3]| spark|         0.0|[5.5,2.5,4.0,1.3]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.5,2.6,4.4,1.2]| spark|         0.0|[5.5,2.6,4.4,1.2]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.5,4.2,1.4,0.2]|hadoop|         1.0|[5.5,4.2,1.4,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[5.6,2.5,3.9,1.1]| spark|         0.0|[5.6,2.5,3.9,1.1]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.6,2.7,4.2,1.3]| spark|         0.0|[5.6,2.7,4.2,1.3]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.6,3.0,4.1,1.3]| spark|         0.0|[5.6,3.0,4.1,1.3]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.7,2.6,3.5,1.0]| spark|         0.0|[5.7,2.6,3.5,1.0]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.8,2.6,4.0,1.2]| spark|         0.0|[5.8,2.6,4.0,1.2]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[5.8,4.0,1.2,0.2]|hadoop|         1.0|[5.8,4.0,1.2,0.2]|[0.0,36.0,0.0]|[0.0,1.0,0.0]|       1.0|        hadoop|
|[6.1,2.6,5.6,1.4]| Scala|         2.0|[6.1,2.6,5.6,1.4]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.2,2.2,4.5,1.5]| spark|         0.0|[6.2,2.2,4.5,1.5]| [0.0,0.0,1.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.2,3.4,5.4,2.3]| Scala|         2.0|[6.2,3.4,5.4,2.3]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.3,2.5,5.0,1.9]| Scala|         2.0|[6.3,2.5,5.0,1.9]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.3,2.8,5.1,1.5]| Scala|         2.0|[6.3,2.8,5.1,1.5]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.4,2.8,5.6,2.1]| Scala|         2.0|[6.4,2.8,5.6,2.1]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.4,2.8,5.6,2.2]| Scala|         2.0|[6.4,2.8,5.6,2.2]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.4,3.2,4.5,1.5]| spark|         0.0|[6.4,3.2,4.5,1.5]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.4,3.2,5.3,2.3]| Scala|         2.0|[6.4,3.2,5.3,2.3]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.5,2.8,4.6,1.5]| spark|         0.0|[6.5,2.8,4.6,1.5]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.6,2.9,4.6,1.3]| spark|         0.0|[6.6,2.9,4.6,1.3]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.6,3.0,4.4,1.4]| spark|         0.0|[6.6,3.0,4.4,1.4]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.8,3.2,5.9,2.3]| Scala|         2.0|[6.8,3.2,5.9,2.3]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[6.9,3.1,4.9,1.5]| spark|         0.0|[6.9,3.1,4.9,1.5]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[6.9,3.2,5.7,2.3]| Scala|         2.0|[6.9,3.2,5.7,2.3]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[7.2,3.0,5.8,1.6]| Scala|         2.0|[7.2,3.0,5.8,1.6]|[29.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|         spark|
|[7.2,3.2,6.0,1.8]| Scala|         2.0|[7.2,3.2,6.0,1.8]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[7.6,3.0,6.6,2.1]| Scala|         2.0|[7.6,3.0,6.6,2.1]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[7.7,3.0,6.1,2.3]| Scala|         2.0|[7.7,3.0,6.1,2.3]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[7.7,3.8,6.7,2.2]| Scala|         2.0|[7.7,3.8,6.7,2.2]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
|[7.9,3.8,6.4,2.0]| Scala|         2.0|[7.9,3.8,6.4,2.0]|[0.0,0.0,31.0]|[0.0,0.0,1.0]|       2.0|         Scala|
+-----------------+------+------------+-----------------+--------------+-------------+----------+--------------+

准确率为: 0.8958333333333334
错误率为: 0.10416666666666663
决策树的结构为: DecisionTreeClassificationModel (uid=dtc_218264842cd2) of depth 5 with 15 nodes
If (feature 2 <= 1.9)
   Predict: 1.0
Else (feature 2 > 1.9)
   If (feature 3 <= 1.7)
    If (feature 0 <= 4.9)
     Predict: 2.0
    Else (feature 0 > 4.9)
     If (feature 1 <= 2.2)
      If (feature 2 <= 4.0)
       Predict: 0.0
      Else (feature 2 > 4.0)
       Predict: 2.0
     Else (feature 1 > 2.2)
      Predict: 0.0
   Else (feature 3 > 1.7)
    If (feature 2 <= 4.8)
     If (feature 0 <= 5.9)
      Predict: 0.0
     Else (feature 0 > 5.9)
      Predict: 2.0
    Else (feature 2 > 4.8)
     Predict: 2.0

Spark 决策树--分类模型的更多相关文章

Spark 决策树--回归模型
package Spark_MLlib import org.apache.spark.ml.Pipeline import org.apache.spark.ml.evaluation.Regres ...
spark 决策树分类算法demo
分类(Classification) 下面的例子说明了怎样导入LIBSVM 数据文件,解析成RDD[LabeledPoint],然后使用决策树进行分类.GINI不纯度作为不纯度衡量标准并且树的最大深度 ...
R语言决策树分类模型
rm(list=ls()) gc() memory.limit(4000) library(corrplot) library(rpart) data_health<-read.csv(&quo ...
Spark学习笔记——构建分类模型
Spark中常见的三种分类模型:线性模型.决策树和朴素贝叶斯模型. 线性模型,简单而且相对容易扩展到非常大的数据集:线性模型又可以分成:1.逻辑回归:2.线性支持向量机决策树是一个强大的非线性技术, ...
Spark机器学习4·分类模型(spark-shell)
线性模型逻辑回归--逻辑损失(logistic loss) 线性支持向量机(Support Vector Machine, SVM)--合页损失(hinge loss) 朴素贝叶斯(Naive Ba ...
笔记︱风控分类模型种类（决策、排序）比较与模型评估体系（ROC/gini/KS/lift）
每每以为攀得众山小,可.每每又切实来到起点,大牛们,缓缓脚步来俺笔记葩分享一下吧,please~ --------------------------- 本笔记源于CDA-DSC课程,由常国珍老师主讲 ...
初识spark的MLP模型
初识Spark的MLP模型 1. MLP介绍 Multi-layer Perceptron(MLP),即多层感知器,是一个前馈式的.具有监督的人工神经网络结构.通过多层感知器可包含多个隐藏层,实现对非 ...
sklearn CART决策树分类
sklearn CART决策树分类决策树是一种常用的机器学习方法,可以用于分类和回归.同时,决策树的训练结果非常容易理解,而且对于数据预处理的要求也不是很高. 理论部分比较经典的决策树是ID3.C ...
ML(4): 决策树分类
决策树(Decision Tree)是用于分类和预测的主要技术,它着眼于从一组无规则的事例推理出决策树表示形式的分类规则,采用自顶向下的递归方式,在决策树的内部节点进行属性值的比较,并根据不同属性判断 ...

随机推荐

json pickle shelve hashlib collections time
import json # Json模块提供了四个功能:dumps.dump.loads.load dic = {'k1':'v1','k2':'v2','k3':'v3'} str_dic = js ...
oracle 互锁的sql查询
SELECT DECODE(request, 0, 'Holder: ', 'Waiter: ') || sid sess, id1, id2, lmode, ...
HDU 1525 Euclid Game
题目大意: 给定2个数a , b,假定b>=a总是从b中取走一个a的整数倍,也就是让 b-k*a(k*a<=b) 每人执行一步这个操作,最后得到0的人胜利结束游戏 (0,a)是一个终止态P ...
CodeForces 159E
题目大意: 给定一堆带颜色和高度的魔方用两种颜色的魔方,一种颜色接一种颜色向上拼接搭建成一个高塔,求高塔的最长高度,以及将拼接的过程中对应的编号顺序输出多种情况成立输出任意一种即可这里首先要对颜 ...
3.2 符号表之二叉查找树BST
一.插入和查找 1.二叉查找树(Binary Search Tree)是一棵二叉树,并且每个结点都含有一个Comparable的键,保证每个结点的键都大于其左子树中任意结点的键而小于其右子树的任意结点 ...
Thinkphp5.0 的请求方式
Thinkphp5.0 的请求方式方法一(使用框架提供的助手函数): public function index(){ $request = request(); dump($request); } ...
Binary Tree Postorder Traversal(各种非递归实现，完美利用栈结构模拟)
1.后序遍历的非递归实现.(左右根) 难点:后序遍历的非递归实现是三种遍历方式中最难的一种.因为在后序遍历中,要保证左孩子和右孩子都已被访问并且左孩子在右孩子前访问才能访问根结点,这就为流程的控制带来 ...
使用idea编译spring-framework5.0源码
自从迈入java开发这个行当,从来没有好好的研究过源码,深感惭愧,话不多说,今天上一篇使用idea编译spring5.0源码. 以下在win中构建和编译过程分为 jdk环境的配置 gradle的下载和 ...
POJ2773 Happy 2006【容斥原理】
题目链接: http://poj.org/problem?id=2773 题目大意: 给你两个整数N和K.找到第k个与N互素的数(互素的数从小到大排列).当中 (1 <= m <= 100 ...
Cocos2D实现上下滚动式状态窗体
大熊猫猪·侯佩原创或翻译作品.欢迎转载,转载请注明出处. 假设认为写的不好请多提意见,假设认为不错请多多支持点赞.谢谢! hopy ;) 有时候要显示的内容太多,我们无法在iOS设备的小屏幕上显示出来 ...

Spark 决策树--分类模型

Spark 决策树--分类模型的更多相关文章

随机推荐

热门专题