spark union issue

 while union we will face issue in mix-match records

var colList = propObject.getProperty("colList").split(",").map(_.trim)

or

1. create df using csv

val df = spark.read.option("header", "true").option("delimiter", "|").option("inferSchema", "true").csv("*")

2. create collist

val collist=df.columns

collist: Array[String] = Array(BU, LEVEL, ranking)

3. select with head and tail method

 val fin=df.select(collist.head,collist.tail:_*).distinct

fin: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [BU: string, LEVEL: string 

4.getting head and tail

val fin=df.select(colList.head,colList.tail:_*).distinct

##########################################

val df2 = spark.read.option("header", "true").option("delimiter", "|").option("inferSchema", "true").csv("*")

val collist=df2.columns

 val fin=df.select(collist.head,collist.tail:_*).distinct

val prch_hist_df = df1.union(df2)