class ShuffleDependency[K, V, C] extends Dependency[Product2[K, V]] with Logging
:: DeveloperApi :: Represents a dependency on the output of a shuffle stage. Note that in the case of shuffle, the RDD is transient since we don't need it on the executor side.
- Annotations
- @DeveloperApi()
- Source
- Dependency.scala
- Alphabetic
- By Inheritance
- ShuffleDependency
- Logging
- Dependency
- Serializable
- Serializable
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
ShuffleDependency(_rdd: RDD[_ <: Product2[K, V]], partitioner: Partitioner, serializer: Serializer = SparkEnv.get.serializer, keyOrdering: Option[Ordering[K]] = None, aggregator: Option[Aggregator[K, V, C]] = None, mapSideCombine: Boolean = false, shuffleWriterProcessor: ShuffleWriteProcessor = new ShuffleWriteProcessor)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[C])
- _rdd
the parent RDD
- partitioner
partitioner used to partition the shuffle output
- serializer
Serializer to use. If not set explicitly then the default serializer, as specified by
spark.serializer
config option, will be used.- keyOrdering
key ordering for RDD's shuffles
- aggregator
map/reduce-side aggregator for RDD's shuffle
- mapSideCombine
whether to perform partial aggregation (also known as map-side combine)
- shuffleWriterProcessor
the processor to control the write behavior in ShuffleMapTask
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- val aggregator: Option[Aggregator[K, V, C]]
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- def getMergerLocs: Seq[BlockManagerId]
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- val keyOrdering: Option[Ordering[K]]
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- val mapSideCombine: Boolean
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def newShuffleMergeState(): Unit
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val partitioner: Partitioner
-
def
rdd: RDD[Product2[K, V]]
- Definition Classes
- ShuffleDependency → Dependency
- val serializer: Serializer
- def setMergerLocs(mergerLocs: Seq[BlockManagerId]): Unit
- val shuffleHandle: ShuffleHandle
- val shuffleId: Int
- def shuffleMergeAllowed: Boolean
- def shuffleMergeEnabled: Boolean
-
def
shuffleMergeFinalized: Boolean
Returns true if push-based shuffle is disabled or if the shuffle merge for this shuffle is finalized.
- def shuffleMergeId: Int
- val shuffleWriterProcessor: ShuffleWriteProcessor
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()