public class EventLoggingListener extends Object implements SparkListener, Logging
Event logging is specified by the following configurable parameters: spark.eventLog.enabled - Whether event logging is enabled. spark.eventLog.compress - Whether to compress logged events spark.eventLog.overwrite - Whether to overwrite any existing files. spark.eventLog.dir - Path to the directory in which events are logged. spark.eventLog.buffer.kb - Buffer size to use when writing to output streams
Constructor and Description |
---|
EventLoggingListener(String appId,
String logBaseDir,
SparkConf sparkConf) |
EventLoggingListener(String appId,
String logBaseDir,
SparkConf sparkConf,
org.apache.hadoop.conf.Configuration hadoopConf) |
Modifier and Type | Method and Description |
---|---|
static String |
APPLICATION_COMPLETE() |
static String |
COMPRESSION_CODEC_PREFIX() |
static String |
DEFAULT_LOG_DIR() |
static String |
getLogDirPath(String logBaseDir,
String appId)
Return a file-system-safe path to the log directory for the given application.
|
static boolean |
isApplicationCompleteFile(String fileName) |
static boolean |
isCompressionCodecFile(String fileName) |
static boolean |
isEventLogFile(String fileName) |
static boolean |
isSparkVersionFile(String fileName) |
static org.apache.hadoop.fs.permission.FsPermission |
LOG_FILE_PERMISSIONS() |
static String |
LOG_PREFIX() |
String |
logDir() |
String |
logDirName() |
scala.collection.mutable.ArrayBuffer<org.json4s.JsonAST.JValue> |
loggedEvents() |
void |
onApplicationEnd(SparkListenerApplicationEnd event)
Called when the application ends
|
void |
onApplicationStart(SparkListenerApplicationStart event)
Called when the application starts
|
void |
onBlockManagerAdded(SparkListenerBlockManagerAdded event)
Called when a new block manager has joined
|
void |
onBlockManagerRemoved(SparkListenerBlockManagerRemoved event)
Called when an existing block manager has been removed
|
void |
onEnvironmentUpdate(SparkListenerEnvironmentUpdate event)
Called when environment properties have been updated
|
void |
onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate event)
Called when the driver receives task metrics from an executor in a heartbeat.
|
void |
onJobEnd(SparkListenerJobEnd event)
Called when a job ends
|
void |
onJobStart(SparkListenerJobStart event)
Called when a job starts
|
void |
onStageCompleted(SparkListenerStageCompleted event)
Called when a stage completes successfully or fails, with information on the completed stage.
|
void |
onStageSubmitted(SparkListenerStageSubmitted event)
Called when a stage is submitted
|
void |
onTaskEnd(SparkListenerTaskEnd event)
Called when a task ends
|
void |
onTaskGettingResult(SparkListenerTaskGettingResult event)
Called when a task begins remotely fetching its result (will not be called for tasks that do
not need to fetch the result remotely).
|
void |
onTaskStart(SparkListenerTaskStart event)
Called when a task starts
|
void |
onUnpersistRDD(SparkListenerUnpersistRDD event)
Called when an RDD is manually unpersisted by the application
|
static String |
parseCompressionCodec(String fileName) |
static EventLoggingInfo |
parseLoggingInfo(org.apache.hadoop.fs.Path logDir,
org.apache.hadoop.fs.FileSystem fileSystem)
Parse the event logging information associated with the logs in the given directory.
|
static EventLoggingInfo |
parseLoggingInfo(String logDir,
org.apache.hadoop.fs.FileSystem fileSystem)
Parse the event logging information associated with the logs in the given directory.
|
static String |
parseSparkVersion(String fileName) |
static String |
SPARK_VERSION_PREFIX() |
void |
start()
Begin logging events.
|
void |
stop()
Stop logging events.
|
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
initializeIfNecessary, initializeLogging, isTraceEnabled, log_, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning
public EventLoggingListener(String appId, String logBaseDir, SparkConf sparkConf, org.apache.hadoop.conf.Configuration hadoopConf)
public EventLoggingListener(String appId, String logBaseDir, SparkConf sparkConf)
public static String DEFAULT_LOG_DIR()
public static String LOG_PREFIX()
public static String SPARK_VERSION_PREFIX()
public static String COMPRESSION_CODEC_PREFIX()
public static String APPLICATION_COMPLETE()
public static org.apache.hadoop.fs.permission.FsPermission LOG_FILE_PERMISSIONS()
public static boolean isEventLogFile(String fileName)
public static boolean isSparkVersionFile(String fileName)
public static boolean isCompressionCodecFile(String fileName)
public static boolean isApplicationCompleteFile(String fileName)
public static String parseSparkVersion(String fileName)
public static String parseCompressionCodec(String fileName)
public static String getLogDirPath(String logBaseDir, String appId)
logBaseDir
- A base directory for the path to the log directory for given application.appId
- A unique app ID.public static EventLoggingInfo parseLoggingInfo(org.apache.hadoop.fs.Path logDir, org.apache.hadoop.fs.FileSystem fileSystem)
Specifically, this looks for event log files, the Spark version file, the compression codec file (if event logs are compressed), and the application completion file (if the application has run to completion).
public static EventLoggingInfo parseLoggingInfo(String logDir, org.apache.hadoop.fs.FileSystem fileSystem)
public String logDir()
public String logDirName()
public scala.collection.mutable.ArrayBuffer<org.json4s.JsonAST.JValue> loggedEvents()
public void start()
public void onStageSubmitted(SparkListenerStageSubmitted event)
SparkListener
onStageSubmitted
in interface SparkListener
public void onTaskStart(SparkListenerTaskStart event)
SparkListener
onTaskStart
in interface SparkListener
public void onTaskGettingResult(SparkListenerTaskGettingResult event)
SparkListener
onTaskGettingResult
in interface SparkListener
public void onTaskEnd(SparkListenerTaskEnd event)
SparkListener
onTaskEnd
in interface SparkListener
public void onEnvironmentUpdate(SparkListenerEnvironmentUpdate event)
SparkListener
onEnvironmentUpdate
in interface SparkListener
public void onStageCompleted(SparkListenerStageCompleted event)
SparkListener
onStageCompleted
in interface SparkListener
public void onJobStart(SparkListenerJobStart event)
SparkListener
onJobStart
in interface SparkListener
public void onJobEnd(SparkListenerJobEnd event)
SparkListener
onJobEnd
in interface SparkListener
public void onBlockManagerAdded(SparkListenerBlockManagerAdded event)
SparkListener
onBlockManagerAdded
in interface SparkListener
public void onBlockManagerRemoved(SparkListenerBlockManagerRemoved event)
SparkListener
onBlockManagerRemoved
in interface SparkListener
public void onUnpersistRDD(SparkListenerUnpersistRDD event)
SparkListener
onUnpersistRDD
in interface SparkListener
public void onApplicationStart(SparkListenerApplicationStart event)
SparkListener
onApplicationStart
in interface SparkListener
public void onApplicationEnd(SparkListenerApplicationEnd event)
SparkListener
onApplicationEnd
in interface SparkListener
public void onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate event)
SparkListener
onExecutorMetricsUpdate
in interface SparkListener
public void stop()