org.apache.spark.sql.avro.IncompatibleSchemaException: Unexpected type org.apache.spark.ml.linalg.VectorUDT code example
Example: org.apache.spark.sql.avro.IncompatibleSchemaException: Unexpected type org.apache.spark.ml.linalg.VectorUDT
import org.apache.spark.sql.functions.udf
import org.apache.spark.sql.functions.col
import org.apache.spark.ml.linalg.Vector
val vectorToArrayUdf = udf((vector: Vector) => vector.toArray)
val output = dataPredictions
.withColumn("probabilities", vectorToArrayUdf(col("probability")))
.select("id", "probabilities", "prediction")
output.write.format("com.databricks.spark.avro").save(path)