pyspark数据准备

鸢尾花数据集

 5.1,3.5,1.4,0.2,Iris-setosa
 4.9,3.0,1.4,0.2,Iris-setosa
 4.7,3.2,1.3,0.2,Iris-setosa
 4.6,3.1,1.5,0.2,Iris-setosa
 5.0,3.6,1.4,0.2,Iris-setosa
 5.4,3.9,1.7,0.4,Iris-setosa
 4.6,3.4,1.4,0.3,Iris-setosa
 5.0,3.4,1.5,0.2,Iris-setosa

转换成libsvm格式代码

 import sys

 file = sys.argv[1]

 def main():
     with open(file,'r') as df:
         for line in df:
             ss = line.strip().split(",")
             if ss[4]=="Iris-setosa":
                 ss[4]=0
             if ss[4]=="Iris-versicolor":
                 ss[4]=1
             if ss[4]=="Iris-virginica":
                 ss[4]=2
             print("%d 1:%.1f 2:%.1f 3:%.1f 4:%.1f"%(ss[4],float(ss[0]),float(ss[1]),float(ss[2]),float(ss[3])))
 if __name__ == '__main__':
     try:
         main()
     except Exception as e:
         raise e

libsvm格式的鸢尾花数据集

 0 1:5.1 2:3.5 3:1.4 4:0.2
 0 1:4.9 2:3.0 3:1.4 4:0.2
 0 1:4.7 2:3.2 3:1.3 4:0.2
 0 1:4.6 2:3.1 3:1.5 4:0.2
 0 1:5.0 2:3.6 3:1.4 4:0.2
 0 1:5.4 2:3.9 3:1.7 4:0.4
 0 1:4.6 2:3.4 3:1.4 4:0.3
 0 1:5.0 2:3.4 3:1.5 4:0.2
 0 1:4.4 2:2.9 3:1.4 4:0.2
 0 1:4.9 2:3.1 3:1.5 4:0.1
 0 1:5.4 2:3.7 3:1.5 4:0.2

pyspark读取libsvm格式数据并转换

>>> from pyspark.mllib.util import MLUtils

>>> examples = MLUtils.loadLibSVMFile(sc, "data/mllib/sample_libsvm_data.txt")

　>>> examples.take(2)
　[Stage 26:>                                                         (0 + 1) / 1]

　[LabeledPoint(0.0, (4,[0,1,2,3],[5.1,3.5,1.4,0.2])), LabeledPoint(0.0, (4,[0,1,2
　,3],[4.9,3.0,1.4,0.2]))]