Package org.apache.spark.util

Class JsonProtocol

Object
org.apache.spark.util.JsonProtocol
public class JsonProtocol extends Object
Serializes SparkListener events to/from JSON. This protocol provides strong backwards- and forwards-compatibility guarantees: any version of Spark should be able to read JSON output written by any other version, including newer versions.

JsonProtocolSuite contains backwards-compatibility tests which check that the current version of JsonProtocol is able to read output written by earlier versions. We do not currently have tests for reading newer JSON output with older Spark versions.

To ensure that we provide these guarantees, follow these rules when modifying these methods:

- Never delete any JSON fields. - Any new JSON fields should be optional; use jsonOption when reading these fields in *FromJson methods.

  • Constructor Details

    • JsonProtocol

      public JsonProtocol()

  • Method Details

    • sparkEventToJsonString

      public static String sparkEventToJsonString(SparkListenerEvent event)
      ------------------------------------------------- * JSON serialization methods for SparkListenerEvents |
      Parameters:
      event - (undocumented)
      Returns:
      (undocumented) --------------------------------------------------

    • sparkEventToJsonString

      public static String sparkEventToJsonString(SparkListenerEvent event, org.apache.spark.util.JsonProtocolOptions options)

    • writeSparkEventToJson

      public static void writeSparkEventToJson(SparkListenerEvent event, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • stageSubmittedToJson

      public static void stageSubmittedToJson(SparkListenerStageSubmitted stageSubmitted, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • stageCompletedToJson

      public static void stageCompletedToJson(SparkListenerStageCompleted stageCompleted, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • taskStartToJson

      public static void taskStartToJson(SparkListenerTaskStart taskStart, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • taskGettingResultToJson

      public static void taskGettingResultToJson(SparkListenerTaskGettingResult taskGettingResult, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • taskEndToJson

      public static void taskEndToJson(SparkListenerTaskEnd taskEnd, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • jobStartToJson

      public static void jobStartToJson(SparkListenerJobStart jobStart, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options)

    • jobEndToJson

      public static void jobEndToJson(SparkListenerJobEnd jobEnd, com.fasterxml.jackson.core.JsonGenerator g)

    • environmentUpdateToJson

      public static void environmentUpdateToJson(SparkListenerEnvironmentUpdate environmentUpdate, com.fasterxml.jackson.core.JsonGenerator g)

    • blockManagerAddedToJson

      public static void blockManagerAddedToJson(SparkListenerBlockManagerAdded blockManagerAdded, com.fasterxml.jackson.core.JsonGenerator g)

    • blockManagerRemovedToJson

      public static void blockManagerRemovedToJson(SparkListenerBlockManagerRemoved blockManagerRemoved, com.fasterxml.jackson.core.JsonGenerator g)

    • unpersistRDDToJson

      public static void unpersistRDDToJson(SparkListenerUnpersistRDD unpersistRDD, com.fasterxml.jackson.core.JsonGenerator g)

    • applicationStartToJson

      public static void applicationStartToJson(SparkListenerApplicationStart applicationStart, com.fasterxml.jackson.core.JsonGenerator g)

    • applicationEndToJson

      public static void applicationEndToJson(SparkListenerApplicationEnd applicationEnd, com.fasterxml.jackson.core.JsonGenerator g)

    • resourceProfileAddedToJson

      public static void resourceProfileAddedToJson(SparkListenerResourceProfileAdded profileAdded, com.fasterxml.jackson.core.JsonGenerator g)

    • executorAddedToJson

      public static void executorAddedToJson(SparkListenerExecutorAdded executorAdded, com.fasterxml.jackson.core.JsonGenerator g)

    • executorRemovedToJson

      public static void executorRemovedToJson(SparkListenerExecutorRemoved executorRemoved, com.fasterxml.jackson.core.JsonGenerator g)

    • logStartToJson

      public static void logStartToJson(SparkListenerLogStart logStart, com.fasterxml.jackson.core.JsonGenerator g)

    • executorMetricsUpdateToJson

      public static void executorMetricsUpdateToJson(SparkListenerExecutorMetricsUpdate metricsUpdate, com.fasterxml.jackson.core.JsonGenerator g)

    • stageExecutorMetricsToJson

      public static void stageExecutorMetricsToJson(SparkListenerStageExecutorMetrics metrics, com.fasterxml.jackson.core.JsonGenerator g)

    • blockUpdateToJson

      public static void blockUpdateToJson(SparkListenerBlockUpdated blockUpdate, com.fasterxml.jackson.core.JsonGenerator g)

    • stageInfoToJson

      public static void stageInfoToJson(StageInfo stageInfo, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options, boolean includeAccumulables)
      ------------------------------------------------------------------- * JSON serialization methods for classes SparkListenerEvents depend on |
      Parameters:
      stageInfo - (undocumented)
      g - (undocumented)
      options - (undocumented)
      includeAccumulables - (undocumented) --------------------------------------------------------------------

    • taskInfoToJson

      public static void taskInfoToJson(TaskInfo taskInfo, com.fasterxml.jackson.core.JsonGenerator g, org.apache.spark.util.JsonProtocolOptions options, boolean includeAccumulables)

    • accumulablesToJson

      public static void accumulablesToJson(scala.collection.Iterable<AccumulableInfo> accumulables, com.fasterxml.jackson.core.JsonGenerator g, boolean includeTaskMetricsAccumulators)

    • accumulableInfoToJson

      public static void accumulableInfoToJson(AccumulableInfo accumulableInfo, com.fasterxml.jackson.core.JsonGenerator g)

    • taskMetricsToJson

      public static void taskMetricsToJson(org.apache.spark.executor.TaskMetrics taskMetrics, com.fasterxml.jackson.core.JsonGenerator g)

    • executorMetricsToJson

      public static void executorMetricsToJson(org.apache.spark.executor.ExecutorMetrics executorMetrics, com.fasterxml.jackson.core.JsonGenerator g)
      Convert executor metrics to JSON.

    • taskEndReasonToJson

      public static void taskEndReasonToJson(TaskEndReason taskEndReason, com.fasterxml.jackson.core.JsonGenerator g)

    • blockManagerIdToJson

      public static void blockManagerIdToJson(BlockManagerId blockManagerId, com.fasterxml.jackson.core.JsonGenerator g)

    • jobResultToJson

      public static void jobResultToJson(JobResult jobResult, com.fasterxml.jackson.core.JsonGenerator g)

    • rddInfoToJson

      public static void rddInfoToJson(RDDInfo rddInfo, com.fasterxml.jackson.core.JsonGenerator g)

    • storageLevelToJson

      public static void storageLevelToJson(StorageLevel storageLevel, com.fasterxml.jackson.core.JsonGenerator g)

    • blockStatusToJson

      public static void blockStatusToJson(BlockStatus blockStatus, com.fasterxml.jackson.core.JsonGenerator g)

    • executorInfoToJson

      public static void executorInfoToJson(ExecutorInfo executorInfo, com.fasterxml.jackson.core.JsonGenerator g)

    • blockUpdatedInfoToJson

      public static void blockUpdatedInfoToJson(BlockUpdatedInfo blockUpdatedInfo, com.fasterxml.jackson.core.JsonGenerator g)

    • executorResourceRequestToJson

      public static void executorResourceRequestToJson(ExecutorResourceRequest execReq, com.fasterxml.jackson.core.JsonGenerator g)

    • executorResourceRequestMapToJson

      public static void executorResourceRequestMapToJson(scala.collection.Map<String,ExecutorResourceRequest> m, com.fasterxml.jackson.core.JsonGenerator g)

    • taskResourceRequestToJson

      public static void taskResourceRequestToJson(TaskResourceRequest taskReq, com.fasterxml.jackson.core.JsonGenerator g)

    • taskResourceRequestMapToJson

      public static void taskResourceRequestMapToJson(scala.collection.Map<String,TaskResourceRequest> m, com.fasterxml.jackson.core.JsonGenerator g)

    • writeMapField

      public static void writeMapField(String name, scala.collection.Map<String,String> m, com.fasterxml.jackson.core.JsonGenerator g)
      ------------------------------ * Util JSON serialization methods |
      Parameters:
      name - (undocumented)
      m - (undocumented)
      g - (undocumented) -------------------------------

    • propertiesToJson

      public static void propertiesToJson(Properties properties, com.fasterxml.jackson.core.JsonGenerator g)

    • UUIDToJson

      public static void UUIDToJson(UUID id, com.fasterxml.jackson.core.JsonGenerator g)

    • stackTraceToJson

      public static void stackTraceToJson(StackTraceElement[] stackTrace, com.fasterxml.jackson.core.JsonGenerator g)

    • exceptionToJson

      public static void exceptionToJson(Exception exception, com.fasterxml.jackson.core.JsonGenerator g)

    • sparkEventFromJson

      public static SparkListenerEvent sparkEventFromJson(String json)

    • sparkEventFromJson

      public static SparkListenerEvent sparkEventFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • stageSubmittedFromJson

      public static SparkListenerStageSubmitted stageSubmittedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • stageCompletedFromJson

      public static SparkListenerStageCompleted stageCompletedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • taskStartFromJson

      public static SparkListenerTaskStart taskStartFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • taskGettingResultFromJson

      public static SparkListenerTaskGettingResult taskGettingResultFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorMetricsFromJson

      public static org.apache.spark.executor.ExecutorMetrics executorMetricsFromJson(com.fasterxml.jackson.databind.JsonNode maybeJson)
      Extract the executor metrics from JSON.

    • taskEndFromJson

      public static SparkListenerTaskEnd taskEndFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • jobStartFromJson

      public static SparkListenerJobStart jobStartFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • jobEndFromJson

      public static SparkListenerJobEnd jobEndFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • resourceProfileAddedFromJson

      public static SparkListenerResourceProfileAdded resourceProfileAddedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorResourceRequestFromJson

      public static ExecutorResourceRequest executorResourceRequestFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • taskResourceRequestFromJson

      public static TaskResourceRequest taskResourceRequestFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • taskResourceRequestMapFromJson

      public static scala.collection.Map<String,TaskResourceRequest> taskResourceRequestMapFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorResourceRequestMapFromJson

      public static scala.collection.Map<String,ExecutorResourceRequest> executorResourceRequestMapFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • environmentUpdateFromJson

      public static SparkListenerEnvironmentUpdate environmentUpdateFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • blockManagerAddedFromJson

      public static SparkListenerBlockManagerAdded blockManagerAddedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • blockManagerRemovedFromJson

      public static SparkListenerBlockManagerRemoved blockManagerRemovedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • unpersistRDDFromJson

      public static SparkListenerUnpersistRDD unpersistRDDFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • applicationStartFromJson

      public static SparkListenerApplicationStart applicationStartFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • applicationEndFromJson

      public static SparkListenerApplicationEnd applicationEndFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorAddedFromJson

      public static SparkListenerExecutorAdded executorAddedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorRemovedFromJson

      public static SparkListenerExecutorRemoved executorRemovedFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • logStartFromJson

      public static SparkListenerLogStart logStartFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorMetricsUpdateFromJson

      public static SparkListenerExecutorMetricsUpdate executorMetricsUpdateFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • stageExecutorMetricsFromJson

      public static SparkListenerStageExecutorMetrics stageExecutorMetricsFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • blockUpdateFromJson

      public static SparkListenerBlockUpdated blockUpdateFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • stageInfoFromJson

      public static StageInfo stageInfoFromJson(com.fasterxml.jackson.databind.JsonNode json)
      --------------------------------------------------------------------- * JSON deserialization methods for classes SparkListenerEvents depend on |
      Parameters:
      json - (undocumented)
      Returns:
      (undocumented) ----------------------------------------------------------------------

    • taskInfoFromJson

      public static TaskInfo taskInfoFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • accumulableInfoFromJson

      public static AccumulableInfo accumulableInfoFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • taskMetricsFromJson

      public static org.apache.spark.executor.TaskMetrics taskMetricsFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • taskEndReasonFromJson

      public static TaskEndReason taskEndReasonFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • blockManagerIdFromJson

      public static BlockManagerId blockManagerIdFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • jobResultFromJson

      public static JobResult jobResultFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • rddInfoFromJson

      public static RDDInfo rddInfoFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • storageLevelFromJson

      public static StorageLevel storageLevelFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • blockStatusFromJson

      public static BlockStatus blockStatusFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • executorInfoFromJson

      public static ExecutorInfo executorInfoFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • blockUpdatedInfoFromJson

      public static BlockUpdatedInfo blockUpdatedInfoFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • resourcesMapFromJson

      public static scala.collection.Map<String,ResourceInformation> resourcesMapFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • mapFromJson

      public static scala.collection.Map<String,String> mapFromJson(com.fasterxml.jackson.databind.JsonNode json)
      -------------------------------- * Util JSON deserialization methods |
      Parameters:
      json - (undocumented)
      Returns:
      (undocumented) ---------------------------------

    • propertiesFromJson

      public static Properties propertiesFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • UUIDFromJson

      public static UUID UUIDFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • stackTraceFromJson

      public static StackTraceElement[] stackTraceFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • exceptionFromJson

      public static Exception exceptionFromJson(com.fasterxml.jackson.databind.JsonNode json)

    • toJsonString

      public static String toJsonString(scala.Function1<com.fasterxml.jackson.core.JsonGenerator,scala.runtime.BoxedUnit> block)