spark MLlib BasicStatistics 统计学基础

一， jar依赖，jsc创建。

package ML.BasicStatistics;

import com.google.common.collect.Lists;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaDoubleRDD;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.DoubleFlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.mllib.linalg.Matrices;
import org.apache.spark.mllib.linalg.Matrix;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.mllib.stat.KernelDensity;
import org.apache.spark.mllib.stat.MultivariateStatisticalSummary;
import org.apache.spark.mllib.stat.Statistics;
import org.apache.spark.mllib.stat.test.ChiSqTestResult;
import org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult;
import org.apache.spark.mllib.util.MLUtils;
import org.apache.spark.rdd.RDD;
import scala.Tuple2;
import scala.runtime.Statics;
import static org.apache.spark.mllib.random.RandomRDDs.*;

import java.util.*;

/**
 * TODO
 *
 * @ClassName: BasicStatistics
 * @author: DingH
 * @since: 2019/4/3 16:11
 */
public class BasicStatistics {
    public static void main(String[] args) {
        System.setProperty("hadoop.home.dir","E:\\hadoop-2.6.5");
        SparkConf conf = new SparkConf().setAppName("BasicStatistics").setMaster("local");
        JavaSparkContext jsc = new JavaSparkContext(conf);

二。Summary statistics

        /**
         * @Title: Statistics.colStats一个实例MultivariateStatisticalSummary，其中包含按列的max，min，mean，variance和非零数，以及总计数
         * Summary statistics：摘要统计
         */
        JavaRDD<Vector> parallelize = jsc.parallelize(Arrays.asList(
                Vectors.dense(1, 0, 3),
                Vectors.dense(2, 0, 4),
                Vectors.dense(3, 0, 5)
        ));
        MultivariateStatisticalSummary summary = Statistics.colStats(parallelize.rdd());
        System.out.println(summary.mean());
        System.out.println(summary.variance());
        System.out.println(summary.numNonzeros());

三。Correlations:相关性

        /**
         * @Title: Correlations:相关性
         */
        JavaRDD<Tuple2<String, String>> parallelize = jsc.parallelize(Lists.newArrayList(
                new Tuple2<String, String>("cat", "11"),
                new Tuple2<String, String>("dog", "22"),
                new Tuple2<String, String>("cat", "33"),
                new Tuple2<String, String>("pig", "44")

        ));

        JavaDoubleRDD seriesX  = parallelize.mapPartitionsToDouble(new DoubleFlatMapFunction<Iterator<Tuple2<String, String>>>() {
            public Iterable<Double> call(Iterator<Tuple2<String, String>> tuple2Iterator) throws Exception {
                ArrayList<Double> strings = new ArrayList<Double>();
                while (tuple2Iterator.hasNext()){
                    strings.add(Double.parseDouble(tuple2Iterator.next()._2));
                }
                return strings;
            }
        });
        JavaDoubleRDD seriesY  = parallelize.mapPartitionsToDouble(new DoubleFlatMapFunction<Iterator<Tuple2<String, String>>>() {
            public Iterable<Double> call(Iterator<Tuple2<String, String>> tuple2Iterator) throws Exception {
                ArrayList<Double> strings = new ArrayList<Double>();
                while (tuple2Iterator.hasNext()){
                    strings.add(Double.parseDouble(tuple2Iterator.next()._2)+1);
                }
                return strings;
            }
        });
         //compute the correlation using Pearson's method. Enter "spearman" for Spearman's method. If a
         //method is not specified, Pearson's method will be used by default.
        double correlation = Statistics.corr(seriesX.srdd(), seriesY.srdd(), "pearson");

        JavaRDD<Vector> parallelize11 = jsc.parallelize(Arrays.asList(
                Vectors.dense(1, 0, 3),
                Vectors.dense(2, 0, 4),
                Vectors.dense(3, 0, 5)
        ));// note that each Vector is a row and not a column
        Matrix correlation2 = Statistics.corr(parallelize11.rdd(), "spearman");
        System.out.println(correlation2);

三，Stratified sampling：分层抽样

        /**
         * @Title: Stratified sampling：分层抽样
         */
        JavaRDD<Tuple2<String, String>> parallelize = jsc.parallelize(Lists.newArrayList(
                new Tuple2<String, String>("cat", "11"),
                new Tuple2<String, String>("dog", "22"),
                new Tuple2<String, String>("cat", "33"),
                new Tuple2<String, String>("pig", "44")

        ));
        JavaPairRDD data = parallelize.mapToPair(new PairFunction<Tuple2<String, String>, String, String>() {
            public Tuple2<String, String> call(Tuple2<String, String> stringStringTuple2) throws Exception {
                return new Tuple2<String, String>(stringStringTuple2._1, stringStringTuple2._2);
            }
        });    // an RDD of any key value pairs
        Map<String, Double> fractions = new HashMap<String, Double>(); // specify the exact fraction desired from each key
        fractions.put("cat",0.5);    //对于每个key取值的概率
        fractions.put("dog",0.8);
        fractions.put("pig",0.8);
        // Get an exact sample from each stratum
        JavaPairRDD approxSample  = data.sampleByKey(false, fractions);
        JavaPairRDD exactSample = data.sampleByKeyExact(false, fractions);
        approxSample.foreach(new VoidFunction() {
            public void call(Object o) throws Exception {
                System.out.println(o);
            }
        });

四。Hypothesis testing  假设检验

        /**
         * @Title: Hypothesis testing  假设检验
         */

        Vector vec = Vectors.dense(1,2,3,4); // a vector composed of the frequencies of events

        // compute the goodness of fit. If a second vector to test against is not supplied as a parameter,
        // the test runs against a uniform distribution.
        ChiSqTestResult goodnessOfFitTestResult = Statistics.chiSqTest(vec);
        // summary of the test including the p-value, degrees of freedom, test statistic, the method used,
        // and the null hypothesis.
        System.out.println(goodnessOfFitTestResult);

        Matrix mat = Matrices.dense(3,2,new double[]{1,2,3,4,5,6}); // a contingency matrix

        // conduct Pearson's independence test on the input contingency matrix
        ChiSqTestResult independenceTestResult = Statistics.chiSqTest(mat);
        // summary of the test including the p-value, degrees of freedom...
        System.out.println(independenceTestResult);

        JavaRDD<LabeledPoint> obs = MLUtils.loadLibSVMFile(jsc.sc(), "/data...").toJavaRDD(); // an RDD of labeled points

        // The contingency table is constructed from the raw (feature, label) pairs and used to conduct
        // the independence test. Returns an array containing the ChiSquaredTestResult for every feature
        // against the label.
        ChiSqTestResult[] featureTestResults = Statistics.chiSqTest(obs.rdd());
        int i = 1;
        for (ChiSqTestResult result : featureTestResults) {
            System.out.println("Column " + i + ":");
            System.out.println(result); // summary of the test
            i++;
        }

        JavaDoubleRDD data = jsc.parallelizeDoubles(Arrays.asList(0.2, 1.0,0.3));
        KolmogorovSmirnovTestResult testResult = Statistics.kolmogorovSmirnovTest(data,"norm");
        // summary of the test including the p-value, test statistic,
        // and null hypothesis
        // if our p-value indicates significance, we can reject the null hypothesis
        System.out.println(testResult);        

五。Random data generation

         /**
         * @Title: Random data generation  :uniform, standard normal, or Poisson.
         */

        JavaDoubleRDD u = normalJavaRDD(jsc, 100,2);
        // Apply a transform to get a random double RDD following `N(1, 4)`.
        JavaRDD<Double> map = u.map(new Function<Double, Double>() {
            public Double call(Double aDouble) throws Exception {
                return 1.0 + 2.0 * aDouble;
            }
        });
        map.foreach(new VoidFunction<Double>() {
            public void call(Double aDouble) throws Exception {
                System.out.println(aDouble);
            }
        });

六。Kernel density estimation

        /**
         * @Title: Kernel density estimation
         */
        JavaRDD<Double> data = jsc.parallelize(Arrays.asList(1.0, 2.0, 3.0));// an RDD of sample data

        // Construct the density estimator with the sample data and a standard deviation for the Gaussian
        // kernels
        KernelDensity kd = new KernelDensity()
          .setSample(data)
          .setBandwidth(3.0);

        // Find density estimates for the given values
        double[] densities = kd.estimate(new double[] {-1.0, 2.0, 5.0});
        for (int i = 0; i < densities.length; i++) {
            System.out.println(densities[i]);
        }