Question

Take first 10 record and last 10 record of file and combine both using DataFrame in Spark?

Answer 1

val loadDF = spark.read.format(“csv”).option(“path”,”file:///home/maria_dev/Files/assignment_table.csv”).load()

val combineDF = loadDF.take(10) ++ loadDF.take(loadDF.count.toInt).takeRight(10)

val schemaDesign = loadDF.first.toSeq.map(c => c.toString.trim).map(col => StructField(col,StringType,true))

val schema = StructType(schemaDesign)

val createDF = spark.createDataFrame(spark.sparkContext.parallelize(combineDF),schema)

createDF.show()