public class WriteAheadLogUtils
extends java.lang.Object
Constructor and Description |
---|
WriteAheadLogUtils() |
Modifier and Type | Method and Description |
---|---|
static WriteAheadLog |
createLogForDriver(SparkConf sparkConf,
java.lang.String fileWalLogDirectory,
org.apache.hadoop.conf.Configuration fileWalHadoopConf)
Create a WriteAheadLog for the driver.
|
static WriteAheadLog |
createLogForReceiver(SparkConf sparkConf,
java.lang.String fileWalLogDirectory,
org.apache.hadoop.conf.Configuration fileWalHadoopConf)
Create a WriteAheadLog for the receiver.
|
static int |
DEFAULT_MAX_FAILURES() |
static int |
DEFAULT_ROLLING_INTERVAL_SECS() |
static java.lang.String |
DRIVER_WAL_BATCHING_CONF_KEY() |
static java.lang.String |
DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY() |
static java.lang.String |
DRIVER_WAL_CLASS_CONF_KEY() |
static java.lang.String |
DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY() |
static java.lang.String |
DRIVER_WAL_MAX_FAILURES_CONF_KEY() |
static java.lang.String |
DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY() |
static boolean |
enableReceiverLog(SparkConf conf) |
static long |
getBatchingTimeout(SparkConf conf)
How long we will wait for the wrappedLog in the BatchedWriteAheadLog to write the records
before we fail the write attempt to unblock receivers.
|
static int |
getMaxFailures(SparkConf conf,
boolean isDriver) |
static int |
getRollingIntervalSecs(SparkConf conf,
boolean isDriver) |
protected static void |
initializeLogIfNecessary(boolean isInterpreter) |
static boolean |
isBatchingEnabled(SparkConf conf,
boolean isDriver) |
protected static boolean |
isTraceEnabled() |
protected static org.slf4j.Logger |
log() |
protected static void |
logDebug(scala.Function0<java.lang.String> msg) |
protected static void |
logDebug(scala.Function0<java.lang.String> msg,
java.lang.Throwable throwable) |
protected static void |
logError(scala.Function0<java.lang.String> msg) |
protected static void |
logError(scala.Function0<java.lang.String> msg,
java.lang.Throwable throwable) |
protected static void |
logInfo(scala.Function0<java.lang.String> msg) |
protected static void |
logInfo(scala.Function0<java.lang.String> msg,
java.lang.Throwable throwable) |
protected static java.lang.String |
logName() |
protected static void |
logTrace(scala.Function0<java.lang.String> msg) |
protected static void |
logTrace(scala.Function0<java.lang.String> msg,
java.lang.Throwable throwable) |
protected static void |
logWarning(scala.Function0<java.lang.String> msg) |
protected static void |
logWarning(scala.Function0<java.lang.String> msg,
java.lang.Throwable throwable) |
static java.lang.String |
RECEIVER_WAL_CLASS_CONF_KEY() |
static java.lang.String |
RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY() |
static java.lang.String |
RECEIVER_WAL_ENABLE_CONF_KEY() |
static java.lang.String |
RECEIVER_WAL_MAX_FAILURES_CONF_KEY() |
static java.lang.String |
RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY() |
static boolean |
shouldCloseFileAfterWrite(SparkConf conf,
boolean isDriver) |
public static java.lang.String RECEIVER_WAL_ENABLE_CONF_KEY()
public static java.lang.String RECEIVER_WAL_CLASS_CONF_KEY()
public static java.lang.String RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY()
public static java.lang.String RECEIVER_WAL_MAX_FAILURES_CONF_KEY()
public static java.lang.String RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY()
public static java.lang.String DRIVER_WAL_CLASS_CONF_KEY()
public static java.lang.String DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY()
public static java.lang.String DRIVER_WAL_MAX_FAILURES_CONF_KEY()
public static java.lang.String DRIVER_WAL_BATCHING_CONF_KEY()
public static java.lang.String DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY()
public static java.lang.String DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY()
public static int DEFAULT_ROLLING_INTERVAL_SECS()
public static int DEFAULT_MAX_FAILURES()
public static boolean enableReceiverLog(SparkConf conf)
public static int getRollingIntervalSecs(SparkConf conf, boolean isDriver)
public static int getMaxFailures(SparkConf conf, boolean isDriver)
public static boolean isBatchingEnabled(SparkConf conf, boolean isDriver)
public static long getBatchingTimeout(SparkConf conf)
conf
- (undocumented)public static boolean shouldCloseFileAfterWrite(SparkConf conf, boolean isDriver)
public static WriteAheadLog createLogForDriver(SparkConf sparkConf, java.lang.String fileWalLogDirectory, org.apache.hadoop.conf.Configuration fileWalHadoopConf)
sparkConf
- (undocumented)fileWalLogDirectory
- (undocumented)fileWalHadoopConf
- (undocumented)public static WriteAheadLog createLogForReceiver(SparkConf sparkConf, java.lang.String fileWalLogDirectory, org.apache.hadoop.conf.Configuration fileWalHadoopConf)
sparkConf
- (undocumented)fileWalLogDirectory
- (undocumented)fileWalHadoopConf
- (undocumented)protected static java.lang.String logName()
protected static org.slf4j.Logger log()
protected static void logInfo(scala.Function0<java.lang.String> msg)
protected static void logDebug(scala.Function0<java.lang.String> msg)
protected static void logTrace(scala.Function0<java.lang.String> msg)
protected static void logWarning(scala.Function0<java.lang.String> msg)
protected static void logError(scala.Function0<java.lang.String> msg)
protected static void logInfo(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void logDebug(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void logTrace(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void logWarning(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void logError(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static boolean isTraceEnabled()
protected static void initializeLogIfNecessary(boolean isInterpreter)