sparkUDF处理复杂嵌套数据结构 map(string, Struct)

发布时间:2024年01月15日
    val spark = SourceUtil.spark
    import spark.implicits._
    val data = spark.sparkContext.parallelize(
      Seq(Row(
        Map(
          "P_MOBILE_NUMBER" -> Row("86_98eesdf3434ffsfsdfs53214010708", 1665295866586L),
          "P_SHA256_MOBILE_NUMBER" -> Row("86_1dsfsdfsdfdeeerrasssdfdfe66fac56e4asdfd07f5ae1", 1665295866586L),
          "OTHER_INFO" -> Row("455sfsdfsfsdd", 1665295866586L)
        )
      ))
    )
  val structType = StructType(Seq(StructField("value", StringType), StructField("version", LongType)))
  val schema = StructType(Seq(StructField("user_params", MapType(StringType, structType))))
  val subSchema = MapType(StringType, structType)

  def formatPhoneNumFromHypersHma = udf((parmas: Map[String, Row]) => {
    parmas.map(userInfo => {
      val name = userInfo._1
      val mobileNumber = userInfo._2.getAs[String]("value").toUpperCase()
      val version = userInfo._2.getAs[Long]("version")
     (name -> Row(mobileNumber, version))
    })
  }, subSchema)

    val df = spark.createDataFrame(data, schema)
    df.printSchema()
    df
      .withColumn("user_params", formatPhoneNumFromHypersHma($"user_params"))
      .show(false)
```
文章来源:https://blog.csdn.net/weixin_46661903/article/details/135601680
本文来自互联网用户投稿,该文观点仅代表作者本人,不代表本站立场。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。