15 Matching Annotations
- Feb 2023
-
stackoverflow.com stackoverflow.com
-
import org.apache.spark.sql._ import org.apache.spark.sql.types._ val rows1 = Seq( Row("1", Row("a", "b"), "8.00", Seq(Row("1","2"), Row("12","22"))), Row("2", Row("c", "d"), "9.00", Seq(Row("3","4"), Row("33","44"))) ) val rows1Rdd = spark.sparkContext.parallelize(rows1, 4) val schema1 = StructType( Seq( StructField("id", StringType, true), StructField("s1", StructType( Seq( StructField("x", StringType, true), StructField("y", StringType, true) ) ), true), StructField("d", StringType, true), StructField("s2", ArrayType(StructType( Seq( StructField("u", StringType, true), StructField("v", StringType, true) ) )), true) ) ) val df1 = spark.createDataFrame(rows1Rdd, schema1)
create a dataframe schema
-
-
stackoverflow.com stackoverflow.com
-
df = df.withColumn( "person", struct( $"person.*", struct( lit("value_1").as("person_field_1"), lit("value_2").as("person_field_2"), ).as("nested_column_within_person") ) )
Example code for adding a complex structure into a nested column
-
- Aug 2021
- Oct 2019
- Sep 2019
- May 2019
- Sep 2017
- Aug 2017
- Jun 2017
-
takwatanabe.me takwatanabe.me
Tags
Annotators
URL
-
- May 2017
-
blog.thehumangeo.com blog.thehumangeo.com
Tags
Annotators
URL
-
- Apr 2017
-
www.datastax.com www.datastax.com
-
stackoverflow.com stackoverflow.com