piyo7 · December 3, 2018 01:06
diff --git a/build.properties b/build.properties
 sbt.version=1.2.6
diff --git a/build.sbt b/build.sbt
 javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
 scalaVersion := "2.12.7"
 libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % "2.4.0",
  "org.apache.spark" %% "spark-sql" % "2.4.0")
diff --git a/file0.scala b/file0.scala
 case class Person(name: String, height: Int)
 val people: Dataset[Person]
diff --git a/file1.scala b/file1.scala
 import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
 val spark = SparkSession.builder().master("local[*]").getOrCreate()
 import spark.implicits._

 case class Person(name: String, height: Int)
 val people: Dataset[Person] = Seq(Person("Sayori", 157), Person("Natsuki", 149), Person("Yuri", 165)).toDS()
diff --git a/file10.txt b/file10.txt
 +-------+------+
 |   name|height|
 +-------+------+
 | Sayori|   157|
 |Natsuki|   149|
 |   Yuri|   165|
 +-------+------+
diff --git a/file11.scala b/file11.scala
 heights.select("name").show()
diff --git a/file12.txt b/file12.txt
 +-------+
 |   name|
 +-------+
 | Sayori|
 |Natsuki|
 |   Yuri|
 +-------+
diff --git a/file13.scala b/file13.scala
 case class Monika(m: Just[Monika], o: Just[Monika], n: Just[Monika], i: Just[Monika], k: Just[Monika], a: Just[Monika])
 people.select("m", "o", "n", "i", "k", "a").as[Monika]
diff --git a/file14.scala b/file14.scala
 implicit class DataSetHelper[T](val dataset: Dataset[T]) extends AnyVal {
  def selectAs[U](implicit encoder: Encoder[U]): Dataset[U] =
      dataset.select(encoder.schema.fields.map(f => new ColumnName(f.name).cast(f.dataType)): _*).as[U]
 }
diff --git a/file15.scala b/file15.scala
 people.selectAs[Monika]
diff --git a/file16.scala b/file16.scala
 val justMonika: Dataset[Person] = people.map(_.copy(name = "Monika"))
diff --git a/file17.scala b/file17.scala
 val justMonika: Dataset[Person] = people.filter(_.name == "Monika")
diff --git a/file2.txt b/file2.txt
 +-------+------+
 |   name|height|
 +-------+------+
 | Sayori|   157|
 |Natsuki|   149|
 |   Yuri|   165|
 +-------+------+
diff --git a/file3.scala b/file3.scala
 val names: DataFrame = people.select("name")
diff --git a/file4.txt b/file4.txt
 +-------+
 |   name|
 +-------+
 | Sayori|
 |Natsuki|
 |   Yuri|
 +-------+
diff --git a/file5.txt b/file5.txt
 [error] (run-main-2) org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];;
 [error] 'Project ['age]
 [error] +- LocalRelation [name#2, height#3]
 [error] org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];;
 [error] 'Project ['age]
 [error] +- LocalRelation [name#2, height#3]
diff --git a/file6.scala b/file6.scala
 case class Height(height: Int)
 val heights: Dataset[Height] = people.as[Height]
diff --git a/file7.scala b/file7.scala
 case class Age(age: Int)
 val ages: Dataset[Age] = people.as[Age]
diff --git a/file8.txt b/file8.txt
 [error] (run-main-0) org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];
 [error] org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];
diff --git a/file9.scala b/file9.scala
 case class Height(height: Int)
 val heights: Dataset[Height] = people.as[Height]
	javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
	scalaVersion := "2.12.7"
	libraryDependencies ++= Seq(
	"org.apache.spark" %% "spark-core" % "2.4.0",
	"org.apache.spark" %% "spark-sql" % "2.4.0")
	case class Person(name: String, height: Int)
	val people: Dataset[Person]
	import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
	val spark = SparkSession.builder().master("local[*]").getOrCreate()
	import spark.implicits._

	case class Person(name: String, height: Int)
	val people: Dataset[Person] = Seq(Person("Sayori", 157), Person("Natsuki", 149), Person("Yuri", 165)).toDS()
	+-------+------+
	\| name\|height\|
	+-------+------+
	\| Sayori\| 157\|
	\|Natsuki\| 149\|
	\| Yuri\| 165\|
	+-------+------+
	case class Monika(m: Just[Monika], o: Just[Monika], n: Just[Monika], i: Just[Monika], k: Just[Monika], a: Just[Monika])
	people.select("m", "o", "n", "i", "k", "a").as[Monika]
	implicit class DataSetHelper[T](val dataset: Dataset[T]) extends AnyVal {
	def selectAs[U](implicit encoder: Encoder[U]): Dataset[U] =
	dataset.select(encoder.schema.fields.map(f => new ColumnName(f.name).cast(f.dataType)): _*).as[U]
	}
	[error] (run-main-2) org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];;
	[error] 'Project ['age]
	[error] +- LocalRelation [name#2, height#3]
	[error] org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];;
	[error] 'Project ['age]
	[error] +- LocalRelation [name#2, height#3]
	case class Height(height: Int)
	val heights: Dataset[Height] = people.as[Height]
	case class Age(age: Int)
	val ages: Dataset[Age] = people.as[Age]
	[error] (run-main-0) org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];
	[error] org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];