public class BlockManager extends Object implements org.apache.spark.network.BlockDataManager, Logging
Note that #initialize() must be called before the BlockManager is usable.
Constructor and Description |
---|
BlockManager(String executorId,
akka.actor.ActorSystem actorSystem,
BlockManagerMaster master,
Serializer defaultSerializer,
long maxMemory,
SparkConf conf,
MapOutputTracker mapOutputTracker,
org.apache.spark.shuffle.ShuffleManager shuffleManager,
org.apache.spark.network.BlockTransferService blockTransferService,
SecurityManager securityManager,
int numUsableCores) |
BlockManager(String execId,
akka.actor.ActorSystem actorSystem,
BlockManagerMaster master,
Serializer serializer,
SparkConf conf,
MapOutputTracker mapOutputTracker,
org.apache.spark.shuffle.ShuffleManager shuffleManager,
org.apache.spark.network.BlockTransferService blockTransferService,
SecurityManager securityManager,
int numUsableCores)
Construct a BlockManager with a memory limit set based on system properties.
|
Modifier and Type | Method and Description |
---|---|
static scala.collection.immutable.Map<BlockId,scala.collection.Seq<BlockManagerId>> |
blockIdsToBlockManagers(BlockId[] blockIds,
SparkEnv env,
BlockManagerMaster blockManagerMaster) |
static scala.collection.immutable.Map<BlockId,scala.collection.Seq<String>> |
blockIdsToExecutorIds(BlockId[] blockIds,
SparkEnv env,
BlockManagerMaster blockManagerMaster) |
static scala.collection.immutable.Map<BlockId,scala.collection.Seq<String>> |
blockIdsToHosts(BlockId[] blockIds,
SparkEnv env,
BlockManagerMaster blockManagerMaster) |
BlockManagerId |
blockManagerId() |
SparkConf |
conf() |
scala.collection.Iterator<Object> |
dataDeserialize(BlockId blockId,
java.nio.ByteBuffer bytes,
Serializer serializer)
Deserializes a ByteBuffer into an iterator of values and disposes of it when the end of
the iterator is reached.
|
java.nio.ByteBuffer |
dataSerialize(BlockId blockId,
scala.collection.Iterator<Object> values,
Serializer serializer)
Serializes into a byte buffer.
|
void |
dataSerializeStream(BlockId blockId,
java.io.OutputStream outputStream,
scala.collection.Iterator<Object> values,
Serializer serializer)
Serializes into a stream.
|
DiskBlockManager |
diskBlockManager() |
DiskStore |
diskStore() |
static void |
dispose(java.nio.ByteBuffer buffer)
Attempt to clean up a ByteBuffer if it is memory-mapped.
|
scala.Option<BlockStatus> |
dropFromMemory(BlockId blockId,
scala.util.Either<Object[],java.nio.ByteBuffer> data)
Drop a block from memory, possibly putting it on disk if applicable.
|
boolean |
externalShuffleServiceEnabled() |
scala.Option<BlockResult> |
get(BlockId blockId)
Get a block from the block manager (either local or remote).
|
org.apache.spark.network.buffer.ManagedBuffer |
getBlockData(BlockId blockId)
Interface to get local block data.
|
BlockObjectWriter |
getDiskWriter(BlockId blockId,
java.io.File file,
Serializer serializer,
int bufferSize,
org.apache.spark.executor.ShuffleWriteMetrics writeMetrics)
A short circuited method to get a block writer that can write data directly to disk.
|
scala.Option<BlockResult> |
getLocal(BlockId blockId)
Get block from local block manager.
|
scala.Option<java.nio.ByteBuffer> |
getLocalBytes(BlockId blockId)
Get block from the local block manager as serialized bytes.
|
scala.collection.Seq<BlockId> |
getMatchingBlockIds(scala.Function1<BlockId,Object> filter)
Get the ids of existing blocks that match the given filter.
|
scala.Option<BlockResult> |
getRemote(BlockId blockId)
Get block from remote block managers.
|
scala.Option<java.nio.ByteBuffer> |
getRemoteBytes(BlockId blockId)
Get block from remote block managers as serialized bytes.
|
scala.Option<Object> |
getSingle(BlockId blockId)
Read a block consisting of a single object.
|
scala.Option<BlockStatus> |
getStatus(BlockId blockId)
Get the BlockStatus for the block identified by the given ID, if it exists.
|
void |
initialize(String appId)
Initializes the BlockManager with the given appId.
|
BlockManagerMaster |
master() |
MemoryStore |
memoryStore() |
scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> |
putArray(BlockId blockId,
Object[] values,
StorageLevel level,
boolean tellMaster,
scala.Option<StorageLevel> effectiveStorageLevel)
Put a new block of values to the block manager.
|
void |
putBlockData(BlockId blockId,
org.apache.spark.network.buffer.ManagedBuffer data,
StorageLevel level)
Put the block locally, using the given storage level.
|
scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> |
putBytes(BlockId blockId,
java.nio.ByteBuffer bytes,
StorageLevel level,
boolean tellMaster,
scala.Option<StorageLevel> effectiveStorageLevel)
Put a new block of serialized bytes to the block manager.
|
scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> |
putIterator(BlockId blockId,
scala.collection.Iterator<Object> values,
StorageLevel level,
boolean tellMaster,
scala.Option<StorageLevel> effectiveStorageLevel) |
scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> |
putSingle(BlockId blockId,
Object value,
StorageLevel level,
boolean tellMaster)
Write a block consisting of a single object.
|
void |
removeBlock(BlockId blockId,
boolean tellMaster)
Remove a block from both memory and disk.
|
int |
removeBroadcast(long broadcastId,
boolean tellMaster)
Remove all blocks belonging to the given broadcast.
|
int |
removeRdd(int rddId)
Remove all blocks belonging to the given RDD.
|
void |
reregister()
Re-register with the master and report all blocks to it.
|
org.apache.spark.network.shuffle.ShuffleClient |
shuffleClient() |
BlockManagerId |
shuffleServerId() |
void |
stop() |
TachyonStore |
tachyonStore() |
void |
waitForAsyncReregister()
For testing.
|
java.io.InputStream |
wrapForCompression(BlockId blockId,
java.io.InputStream s)
Wrap an input stream for compression if block compression is enabled for its block type
|
java.io.OutputStream |
wrapForCompression(BlockId blockId,
java.io.OutputStream s)
Wrap an output stream for compression if block compression is enabled for its block type
|
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
initializeIfNecessary, initializeLogging, isTraceEnabled, log_, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning
public BlockManager(String executorId, akka.actor.ActorSystem actorSystem, BlockManagerMaster master, Serializer defaultSerializer, long maxMemory, SparkConf conf, MapOutputTracker mapOutputTracker, org.apache.spark.shuffle.ShuffleManager shuffleManager, org.apache.spark.network.BlockTransferService blockTransferService, SecurityManager securityManager, int numUsableCores)
public BlockManager(String execId, akka.actor.ActorSystem actorSystem, BlockManagerMaster master, Serializer serializer, SparkConf conf, MapOutputTracker mapOutputTracker, org.apache.spark.shuffle.ShuffleManager shuffleManager, org.apache.spark.network.BlockTransferService blockTransferService, SecurityManager securityManager, int numUsableCores)
public static void dispose(java.nio.ByteBuffer buffer)
public static scala.collection.immutable.Map<BlockId,scala.collection.Seq<BlockManagerId>> blockIdsToBlockManagers(BlockId[] blockIds, SparkEnv env, BlockManagerMaster blockManagerMaster)
public static scala.collection.immutable.Map<BlockId,scala.collection.Seq<String>> blockIdsToExecutorIds(BlockId[] blockIds, SparkEnv env, BlockManagerMaster blockManagerMaster)
public static scala.collection.immutable.Map<BlockId,scala.collection.Seq<String>> blockIdsToHosts(BlockId[] blockIds, SparkEnv env, BlockManagerMaster blockManagerMaster)
public BlockManagerMaster master()
public SparkConf conf()
public DiskBlockManager diskBlockManager()
public MemoryStore memoryStore()
public DiskStore diskStore()
public TachyonStore tachyonStore()
public boolean externalShuffleServiceEnabled()
public BlockManagerId blockManagerId()
public BlockManagerId shuffleServerId()
public org.apache.spark.network.shuffle.ShuffleClient shuffleClient()
public void initialize(String appId)
This method initializes the BlockTransferService and ShuffleClient, registers with the BlockManagerMaster, starts the BlockManagerWorker actor, and registers with a local shuffle service if configured.
public void reregister()
Note that this method must be called without any BlockInfo locks held.
public void waitForAsyncReregister()
public org.apache.spark.network.buffer.ManagedBuffer getBlockData(BlockId blockId)
getBlockData
in interface org.apache.spark.network.BlockDataManager
public void putBlockData(BlockId blockId, org.apache.spark.network.buffer.ManagedBuffer data, StorageLevel level)
putBlockData
in interface org.apache.spark.network.BlockDataManager
public scala.Option<BlockStatus> getStatus(BlockId blockId)
public scala.collection.Seq<BlockId> getMatchingBlockIds(scala.Function1<BlockId,Object> filter)
public scala.Option<BlockResult> getLocal(BlockId blockId)
public scala.Option<java.nio.ByteBuffer> getLocalBytes(BlockId blockId)
public scala.Option<BlockResult> getRemote(BlockId blockId)
public scala.Option<java.nio.ByteBuffer> getRemoteBytes(BlockId blockId)
public scala.Option<BlockResult> get(BlockId blockId)
public scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> putIterator(BlockId blockId, scala.collection.Iterator<Object> values, StorageLevel level, boolean tellMaster, scala.Option<StorageLevel> effectiveStorageLevel)
public BlockObjectWriter getDiskWriter(BlockId blockId, java.io.File file, Serializer serializer, int bufferSize, org.apache.spark.executor.ShuffleWriteMetrics writeMetrics)
public scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> putArray(BlockId blockId, Object[] values, StorageLevel level, boolean tellMaster, scala.Option<StorageLevel> effectiveStorageLevel)
public scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> putBytes(BlockId blockId, java.nio.ByteBuffer bytes, StorageLevel level, boolean tellMaster, scala.Option<StorageLevel> effectiveStorageLevel)
public scala.Option<Object> getSingle(BlockId blockId)
public scala.collection.Seq<scala.Tuple2<BlockId,BlockStatus>> putSingle(BlockId blockId, Object value, StorageLevel level, boolean tellMaster)
public scala.Option<BlockStatus> dropFromMemory(BlockId blockId, scala.util.Either<Object[],java.nio.ByteBuffer> data)
Return the block status if the given block has been updated, else None.
public int removeRdd(int rddId)
public int removeBroadcast(long broadcastId, boolean tellMaster)
public void removeBlock(BlockId blockId, boolean tellMaster)
public java.io.OutputStream wrapForCompression(BlockId blockId, java.io.OutputStream s)
public java.io.InputStream wrapForCompression(BlockId blockId, java.io.InputStream s)
public void dataSerializeStream(BlockId blockId, java.io.OutputStream outputStream, scala.collection.Iterator<Object> values, Serializer serializer)
public java.nio.ByteBuffer dataSerialize(BlockId blockId, scala.collection.Iterator<Object> values, Serializer serializer)
public scala.collection.Iterator<Object> dataDeserialize(BlockId blockId, java.nio.ByteBuffer bytes, Serializer serializer)
public void stop()