public class CheckpointRDD<T> extends RDD<T>
Constructor and Description |
---|
CheckpointRDD(SparkContext sc,
String checkpointPath,
scala.reflect.ClassTag<T> evidence$1) |
Modifier and Type | Method and Description |
---|---|
Broadcast<SerializableWritable<org.apache.hadoop.conf.Configuration>> |
broadcastedConf() |
void |
checkpoint()
Mark this RDD for checkpointing.
|
String |
checkpointPath() |
scala.collection.Iterator<T> |
compute(Partition split,
TaskContext context)
:: DeveloperApi ::
Implemented by subclasses to compute a given partition.
|
org.apache.hadoop.fs.FileSystem |
fs() |
Partition[] |
getPartitions()
Implemented by subclasses to return the set of partitions in this RDD.
|
scala.collection.Seq<String> |
getPreferredLocations(Partition split)
Optionally overridden by subclasses to specify placement preferences.
|
static void |
main(String[] args) |
static <T> scala.collection.Iterator<T> |
readFromFile(org.apache.hadoop.fs.Path path,
Broadcast<SerializableWritable<org.apache.hadoop.conf.Configuration>> broadcastedConf,
TaskContext context) |
static String |
splitIdToFile(int splitId) |
static <T> void |
writeToFile(String path,
Broadcast<SerializableWritable<org.apache.hadoop.conf.Configuration>> broadcastedConf,
int blockSize,
TaskContext ctx,
scala.collection.Iterator<T> iterator,
scala.reflect.ClassTag<T> evidence$2) |
aggregate, cache, cartesian, checkpointData, coalesce, collect, collect, collectPartitions, computeOrReadCheckpoint, conf, context, count, countApprox, countApproxDistinct, countApproxDistinct, countByValue, countByValueApprox, creationSite, dependencies, distinct, distinct, doCheckpoint, elementClassTag, filter, filterWith, first, flatMap, flatMapWith, fold, foreach, foreachPartition, foreachWith, getCheckpointFile, getCreationSite, getNarrowAncestors, getStorageLevel, glom, groupBy, groupBy, groupBy, id, intersection, intersection, intersection, isCheckpointed, iterator, keyBy, map, mapPartitions, mapPartitionsWithContext, mapPartitionsWithIndex, mapPartitionsWithSplit, mapWith, markCheckpointed, max, min, name, partitioner, partitions, persist, persist, pipe, pipe, pipe, preferredLocations, randomSplit, reduce, repartition, retag, retag, sample, saveAsObjectFile, saveAsTextFile, saveAsTextFile, setName, sortBy, sparkContext, subtract, subtract, subtract, take, takeOrdered, takeSample, toArray, toDebugString, toJavaRDD, toLocalIterator, top, toString, union, unpersist, zip, zipPartitions, zipPartitions, zipPartitions, zipPartitions, zipPartitions, zipPartitions, zipWithIndex, zipWithUniqueId
initializeIfNecessary, initializeLogging, isTraceEnabled, log_, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning
public CheckpointRDD(SparkContext sc, String checkpointPath, scala.reflect.ClassTag<T> evidence$1)
public static String splitIdToFile(int splitId)
public static <T> void writeToFile(String path, Broadcast<SerializableWritable<org.apache.hadoop.conf.Configuration>> broadcastedConf, int blockSize, TaskContext ctx, scala.collection.Iterator<T> iterator, scala.reflect.ClassTag<T> evidence$2)
public static <T> scala.collection.Iterator<T> readFromFile(org.apache.hadoop.fs.Path path, Broadcast<SerializableWritable<org.apache.hadoop.conf.Configuration>> broadcastedConf, TaskContext context)
public static void main(String[] args)
public String checkpointPath()
public Broadcast<SerializableWritable<org.apache.hadoop.conf.Configuration>> broadcastedConf()
public org.apache.hadoop.fs.FileSystem fs()
public Partition[] getPartitions()
RDD
public scala.collection.Seq<String> getPreferredLocations(Partition split)
RDD
public scala.collection.Iterator<T> compute(Partition split, TaskContext context)
RDD
public void checkpoint()
RDD
checkpoint
in class RDD<T>