java实现spark常用算子之countbykey

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

/**
 *  countbykey 算子:
 *  只能用在(K,V)结构。
 *  用来统计每个key有多少个，返回一个（K,int）
 */
public class CountByKeyOperator {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setMaster("local").setAppName("countbykey");
        JavaSparkContext sc = new JavaSparkContext(conf);
        List<Tuple2<String,String>> names = Arrays.asList(
                new Tuple2<String,String>("w1","1"),
                new Tuple2<String,String>("w2","1"),
                new Tuple2<String,String>("w1","1"),
                new Tuple2<String,String>("w3","1"),
                new Tuple2<String,String>("w2","1")
        );

        JavaPairRDD<String,String> nameRdd = sc.parallelizePairs(names);
        Map<String,Long> reusult = nameRdd.countByKey();

        for(Map.Entry<String,Long> map: reusult.entrySet()) {
            System.err.println(map.getKey()+":"+map.getValue());
        }

    }
}

微信扫描下图二维码加入博主知识星球,获取更多大数据、人工智能、算法等免费学习资料哦！