UTFDataFormatException

https://stackoverflow.com/questions/41505599/task-not-serializable-in-spark-caused-by-utfdataformatexception-encoded-string

package com.gac.xs6.common.conf

import com.google.inject.Singleton
import com.typesafe.config.{Config, ConfigFactory}
import com.github.nscala_time.time.Imports._

/**
  * HDFS 路径配置
  */
@Singleton
class PathsConfiguration extends Serializable {
  private val config:     Config = ConfigFactory.load()
  val yesterday:          String = DateTime.yesterday.toString("yyyyMMdd")
  lazy val pathConfig:    Config = config.getConfig("hdfs")
  lazy val nationA2APHEV: String = pathConfig.getString("nationA2APHEV") ++ "/d="  ++ yesterday
  lazy val nationA5HEV:   String = pathConfig.getString("nationA5HEV")   ++ "/d="  ++ yesterday
  lazy val nationA51:     String = pathConfig.getString("nationA51")     ++ "/d="  ++ yesterday
  lazy val nationA75:     String = pathConfig.getString("nationA75")     ++ "/d="  ++ yesterday
  lazy val enterpriseA75: String = pathConfig.getString("enterpriseA75") ++ "/dt=" ++ yesterday
}

9/07/05 18:57:20 org.apache.spark.internal.Logging$class.logError(Logging.scala:91) ERROR Utils: Exception encountered
java.io.UTFDataFormatException: encoded string too long: 73484 bytes
    at java.io.DataOutputStream.writeUTF(DataOutputStream.java:364)
    at java.io.DataOutputStream.writeUTF(DataOutputStream.java:323)
    at com.typesafe.config.impl.SerializedConfigValue.writeValueData(SerializedConfigValue.java:314)
    at com.typesafe.config.impl.SerializedConfigValue.writeValue(SerializedConfigValue.java:388)
    at com.typesafe.config.impl.SerializedConfigValue.writeValueData(SerializedConfigValue.java:328)
    at com.typesafe.config.impl.SerializedConfigValue.writeValue(SerializedConfigValue.java:388)
    at com.typesafe.config.impl.SerializedConfigValue.writeValueData(SerializedConfigValue.java:328)
    at com.typesafe.config.impl.SerializedConfigValue.writeValue(SerializedConfigValue.java:388)
    at com.typesafe.config.impl.SerializedConfigValue.writeValueData(SerializedConfigValue.java:328)
    at com.typesafe.config.impl.SerializedConfigValue.writeValue(SerializedConfigValue.java:388)
    at com.typesafe.config.impl.SerializedConfigValue.writeExternal(SerializedConfigValue.java:454)

我传了一个 TypeSafeConfig 配置对象(已序列化过), 可能这个参数超出了 64K?

当我把函数中传递的 Config 参数放到函数本身中后,就不报这个错误了。

最后编辑于
©著作权归作者所有,转载或内容合作请联系作者
平台声明:文章内容(如有图片或视频亦包括在内)由作者上传并发布,文章内容仅代表作者本人观点,简书系信息发布平台,仅提供信息存储服务。