DataFrame转Dataset

DataFrame 转 Dataset，DataFrame要指定类型，然后通过 as[] 转成Dataset

案例

样例类

package rdd2dataframe

import java.sql.Timestamp

//定义样例类，封装数据
case class Cdz(id: String, eleNum: Integer, time:Integer, startTime: Timestamp, endTime: Timestamp) {

}

主类

package rdd2dataframe

import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
import org.apache.spark.{SparkConf, SparkContext}

object DataFrame2Dataset {
  def main(args: Array[String]): Unit = {

    // 建立和Spark框架的连接
    val sparConf = new SparkConf().setMaster("local[*]").setAppName("sparkSQL")
    val sc = new SparkContext(sparConf)
    //创建SparkSession
    val spark: SparkSession = SparkSession.builder().config(sparConf).getOrCreate()

    // 读取文本文件，获取一行一行的数据
    val df: DataFrame = spark.read
      .option("header", "false") //默认为false，表示第一行不是表头
      .option("sep", ",") //列分隔符
      .schema(ScalaReflection.schemaFor[Cdz].dataType.asInstanceOf[StructType])
      .csv("cdz.txt") //读取文件

    import spark.implicits._
    val ds: Dataset[Cdz] = df.as[Cdz]
    ds.printSchema()
    ds.show()
    spark.close()
  }
}

Dataset转DataFrame

Dataset 通过 .toDF() 转DataFrame

例子

package rdd2dataframe

import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
import org.apache.spark.{SparkConf, SparkContext}

object Dataset2DataFrame {
  def main(args: Array[String]): Unit = {

    // 建立和Spark框架的连接
    val sparConf = new SparkConf().setMaster("local[*]").setAppName("sparkSQL")
    val sc = new SparkContext(sparConf)
    //创建SparkSession
    val spark: SparkSession = SparkSession.builder().config(sparConf).getOrCreate()

    // 读取文本文件，获取一行一行的数据
    val df: DataFrame = spark.read
      .option("header", "false") //默认为false，表示第一行不是表头
      .option("sep", ",") //列分隔符
      .schema(ScalaReflection.schemaFor[Cdz].dataType.asInstanceOf[StructType])
      .csv("cdz.txt") //读取文件

    import spark.implicits._
    val ds: Dataset[Cdz] = df.as[Cdz]

    // dataset 转 dataframe
    val df2: DataFrame = ds.toDF()
    df2.printSchema()
    df2.show()
    spark.close()
  }
}

原文出处：/show_1IX4K7xbNaLU.html