sparksql dataset

java

 /**
  *2.0之后使用sparksession即可,不需要再去创建sqlcontext
  *@author Tele
  *
  */
 public class Demo {
     private static SparkConf conf = new SparkConf().setAppName("dataframedemo").setMaster("local");
     private static JavaSparkContext jsc = new JavaSparkContext(conf);

     private static SparkSession session = new SparkSession(jsc.sc());

     //创建sparksession(可以不创建jsc)
 //    private static SparkSession session = SparkSession.builder().appName("asd").master("local").getOrCreate();

 //    private static SQLContext sqlContext = new SQLContext(session);

     public static void main(String[] args) {

         //在java中返回是dataset,在scala中返回的是dataframe
     //    sqlContext.read().json("xx")

         Dataset<Row> dataset = session.read().json("./src/main/java/cn/tele/spark_sql/dataframe/students.json");
         //输出全部数据
         dataset.show();

         //输出某一列
         dataset.select("id").show();
         dataset.select(dataset.col("id")).show();

         //判断id>2
         dataset.select(dataset.col("id").gt(2)).show();

         //判断id>=2
         dataset.select(dataset.col("id").geq(2)).show();

         //将age += 100
         dataset.select(dataset.col("age").plus(100)).show();

         //输出元数据
         dataset.printSchema();

         jsc.close();

     }
 }

scala

 object Demo {
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf().setAppName("demo").setMaster("local")
     val sc = new SparkContext(conf)

     val sqlcontext = new SQLContext(sc)

     val dataframe = sqlcontext.read.json("./src/main/scala/cn/tele/spark_sql/dataframe/students.json")

     dataframe.select(dataframe.col("id")).show()

     dataframe.select(dataframe.col("id").plus(1)).show()

     dataframe.select(dataframe.col("id") + 1).show()

     dataframe.select("id","name").show()

     dataframe.select(dataframe.col("id").geq("2")).show()

     dataframe.printSchema()

   }
 }