sparkR处理Gb级数据集

spark集群搭建及介绍：敬请关注

数据集：http://pan.baidu.com/s/1sjYN7lF

总结：使用sparkR进行数据分析建模相比R大致有3-5倍的提升

查看原始数据集：通过iris数据集生成

[root@master data]#pwd

/data

[root@master data]#ls -lhsrt iris1g.txt

1.3G -rw-r--r-- 1root root 1.3G Feb 16 14:16 iris1g.txt

登录sparkR：

sparkR --masteryarn-client  --num-executors 15

#1、载入数据：47671650千万数据，耗时1.60118mins

> (time1 <-Sys.time())

[1] "2016-02-1810:04:08 CST"

> data_iris <-read.table("/data/iris1g.txt", stringsAsFactors=T, sep=",",header=T, comment="", quote=NULL, encoding="UTF-8")

> Sys.time() -time1

Time difference of1.60118 mins

#使用data.table中的fread读取数据:4000千万数据，耗时1.910114
mins

library(data.table)

(time1 <-Sys.time())

data_iris <- fread("D:\\R大数据集/iris1g.txt",stringsAsFactors=T, sep=",",
header=T, encoding="UTF-8")

Sys.time() - time1

#2、数据预处理

> dim(data_iris)

[1] 47671650        5

str(data_iris)

> str(data_iris)

'data.frame':        47671650obs. of  5 variables:

$ X.Sepal.Length.: num  5.1 4.9 4.7 4.6 5 5.4 4.6 5 4.4 4.9 ...

$ X.Sepal.Width. : num  3.5 3 3.2 3.1 3.6 3.9 3.4 3.4 2.9 3.1 ...

$ X.Petal.Length.: num  1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 ...

$ X.Petal.Width. : num  0.2 0.2 0.2 0.2 0.2 0.4 0.3 0.2 0.2 0.1 ...

$ X.Species.     : Factor w/ 3 levels"\"setosa\"","\"versicolor\"",..: 1 1 11 1 1 1 1 1 1 ...

> names(iris)

[1]"Sepal.Length" "Sepal.Width"  "Petal.Length""Petal.Width" "Species"

>names(data_iris)

[1]"X.Sepal.Length." "X.Sepal.Width."  "X.Petal.Length.""X.Petal.Width."

[5]"X.Species."

>

>names(data_iris) <- names(iris)

>names(data_iris)

[1]"Sepal.Length" "Sepal.Width"  "Petal.Length""Petal.Width" "Species"

#3、创建训练集和測试集数据

library(caret)

#创建训练集和測试集数据：耗时6.402254 secs

> (time1 <-Sys.time())

[1] "2016-02-1810:10:35 CST"

> ind <-base:::sample(3, nrow(data_iris), prob=c(0.3, 0.2, 0.5), replace=T)

> train <-data_iris[ind==1, ]

> test <-data_iris[ind==2, ]

> Sys.time() -time1

Time difference of6.402254 secs

#使用createDataPartition导致内存溢出

#(time1 <-Sys.time())

#index <-createDataPartition(data$Species, nrow(data), p=0.7, list=F)

#Sys.time() - time1

#train <-data[index, ]

#test <-data[-index, ]

> dim(train)

[1] 14301827        5

> dim(test)

[1] 9533737       5

memory.size()

gc()

#4、建模

#1）随机森林

#library(randomForest)

#model <-randomForest(train$X.Species.~., data=train, ntree=50, nPerm=10, mtry=3,proximity=T, importance=T)

#随机森林建模导致内存溢出

#2）使用决策时间建模:1.891634
mins

library(party)

> (time1 <-Sys.time())

[1] "2016-02-1810:12:08 CST"

> model <-ctree(Species~., data=train)

> Sys.time() -time1

Time difference of

>print(object.size(model), units="Mb")

6372.7 Mb

#str(model)

> summary(model)

Length     Class       Mode

1 BinaryTree         S4

#5、预測

> (time1 <-Sys.time())

[1] "2016-02-1810:14:49 CST"

> pred <-predict(model, test)

> Sys.time() -time1

Time difference of36.58139 secs

#6、模型评估

table(pred,test$Species)

>mean(pred==test$Species)

[1] 1

>base:::table(pred, test$Species)

pred           "setosa""versicolor" "virginica"

"setosa"      3177256            0           0

"versicolor"        0     3178471           0

"virginica"         0            0     3178010

>library(gmodels)

>CrossTable(pred, test$Species)

Cell Contents

|-------------------------|

|                       N |

| Chi-squarecontribution |

|           N / Row Total |

|           N / Col Total |

|         N / Table Total |

|-------------------------|

Total Observationsin Table:  9533737

| test$Species

pred |     "setosa" |"versicolor" | "virginica" |    RowTotal |

-------------|--------------|--------------|--------------|--------------|

"setosa" |      3177256 |            0 |            0 |      3177256 |

| 4238091.601 |  1059271.517 |  1059117.882 |              |

|        1.000 |        0.000 |        0.000 |        0.333 |

|        1.000 |        0.000 |        0.000 |              |

|        0.333 |        0.000 |        0.000 |              |

-------------|--------------|--------------|--------------|--------------|

"versicolor"|            0 |      3178471 |            0 |      3178471 |

| 1059271.517 |  4236471.588 |  1059522.895 |              |

|        0.000 |        1.000 |        0.000 |        0.333 |

|        0.000 |        1.000 |        0.000 |              |

|        0.000 |        0.333 |        0.000 |              |

-------------|--------------|--------------|--------------|--------------|

"virginica" |            0 |            0 |      3178010 |      3178010 |

| 1059117.882 |  1059522.895 |  4237086.223 |              |

|        0.000 |        0.000 |        1.000 |        0.333 |

|        0.000 |        0.000 |        1.000 |              |

|        0.000 |        0.000 |        0.333 |              |

-------------|--------------|--------------|--------------|--------------|

Column Total |      3177256 |      3178471 |      3178010 |      9533737 |

|        0.333 |        0.333 |        0.333 |              |

-------------|--------------|--------------|--------------|--------------|

```