org.apache.spark.scheduler
Class StatsReportListener

Object
  extended by org.apache.spark.scheduler.StatsReportListener
All Implemented Interfaces:
Logging, SparkListener

public class StatsReportListener
extends Object
implements SparkListener, Logging

:: DeveloperApi :: Simple SparkListener that logs a few summary statistics when each stage completes


Constructor Summary
StatsReportListener()
           
 
Method Summary
static scala.Option<org.apache.spark.util.Distribution> extractDoubleDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics, scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)
           
static scala.Option<org.apache.spark.util.Distribution> extractLongDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics, scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)
           
static long hours()
           
static String millisToString(long ms)
          Reformat a time interval in milliseconds to a prettier format for output
static long minutes()
           
 void onStageCompleted(SparkListenerStageCompleted stageCompleted)
          Called when a stage completes successfully or fails, with information on the completed stage.
 void onTaskEnd(SparkListenerTaskEnd taskEnd)
          Called when a task ends
static int[] percentiles()
           
static String percentilesHeader()
           
static double[] probabilities()
           
static long seconds()
           
static void showBytesDistribution(String heading, scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric, scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
           
static void showBytesDistribution(String heading, scala.Option<org.apache.spark.util.Distribution> dOpt)
           
static void showBytesDistribution(String heading, org.apache.spark.util.Distribution dist)
           
static void showDistribution(String heading, scala.Option<org.apache.spark.util.Distribution> dOpt, scala.Function1<Object,String> formatNumber)
           
static void showDistribution(String heading, scala.Option<org.apache.spark.util.Distribution> dOpt, String format)
           
static void showDistribution(String heading, org.apache.spark.util.Distribution d, scala.Function1<Object,String> formatNumber)
           
static void showDistribution(String heading, String format, scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric, scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
           
static void showMillisDistribution(String heading, scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric, scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
           
static void showMillisDistribution(String heading, scala.Option<org.apache.spark.util.Distribution> dOpt)
           
 
Methods inherited from class Object
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 
Methods inherited from interface org.apache.spark.scheduler.SparkListener
onApplicationEnd, onApplicationStart, onBlockManagerAdded, onBlockManagerRemoved, onEnvironmentUpdate, onExecutorAdded, onExecutorMetricsUpdate, onExecutorRemoved, onJobEnd, onJobStart, onStageSubmitted, onTaskGettingResult, onTaskStart, onUnpersistRDD
 
Methods inherited from interface org.apache.spark.Logging
initializeIfNecessary, initializeLogging, isTraceEnabled, log_, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning
 

Constructor Detail

StatsReportListener

public StatsReportListener()
Method Detail

percentiles

public static int[] percentiles()

probabilities

public static double[] probabilities()

percentilesHeader

public static String percentilesHeader()

extractDoubleDistribution

public static scala.Option<org.apache.spark.util.Distribution> extractDoubleDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics,
                                                                                         scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)

extractLongDistribution

public static scala.Option<org.apache.spark.util.Distribution> extractLongDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics,
                                                                                       scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)

showDistribution

public static void showDistribution(String heading,
                                    org.apache.spark.util.Distribution d,
                                    scala.Function1<Object,String> formatNumber)

showDistribution

public static void showDistribution(String heading,
                                    scala.Option<org.apache.spark.util.Distribution> dOpt,
                                    scala.Function1<Object,String> formatNumber)

showDistribution

public static void showDistribution(String heading,
                                    scala.Option<org.apache.spark.util.Distribution> dOpt,
                                    String format)

showDistribution

public static void showDistribution(String heading,
                                    String format,
                                    scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
                                    scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)

showBytesDistribution

public static void showBytesDistribution(String heading,
                                         scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
                                         scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)

showBytesDistribution

public static void showBytesDistribution(String heading,
                                         scala.Option<org.apache.spark.util.Distribution> dOpt)

showBytesDistribution

public static void showBytesDistribution(String heading,
                                         org.apache.spark.util.Distribution dist)

showMillisDistribution

public static void showMillisDistribution(String heading,
                                          scala.Option<org.apache.spark.util.Distribution> dOpt)

showMillisDistribution

public static void showMillisDistribution(String heading,
                                          scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
                                          scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)

seconds

public static long seconds()

minutes

public static long minutes()

hours

public static long hours()

millisToString

public static String millisToString(long ms)
Reformat a time interval in milliseconds to a prettier format for output

Parameters:
ms - (undocumented)
Returns:
(undocumented)

onTaskEnd

public void onTaskEnd(SparkListenerTaskEnd taskEnd)
Description copied from interface: SparkListener
Called when a task ends

Specified by:
onTaskEnd in interface SparkListener
Parameters:
taskEnd - (undocumented)

onStageCompleted

public void onStageCompleted(SparkListenerStageCompleted stageCompleted)
Description copied from interface: SparkListener
Called when a stage completes successfully or fails, with information on the completed stage.

Specified by:
onStageCompleted in interface SparkListener
Parameters:
stageCompleted - (undocumented)