class ShuffleDependency[K, V, C] extends Dependency[Product2[K, V]] with Logging
:: DeveloperApi :: Represents a dependency on the output of a shuffle stage. Note that in the case of shuffle, the RDD is transient since we don't need it on the executor side.
- Annotations
- @DeveloperApi()
- Source
- Dependency.scala
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- ShuffleDependency
- Logging
- Dependency
- Serializable
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- Protected
Instance Constructors
- new ShuffleDependency(rdd: RDD[_ <: Product2[K, V]], partitioner: Partitioner, serializer: Serializer, keyOrdering: Option[Ordering[K]], aggregator: Option[Aggregator[K, V, C]], mapSideCombine: Boolean, shuffleWriterProcessor: ShuffleWriteProcessor)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[C])
- new ShuffleDependency(_rdd: RDD[_ <: Product2[K, V]], partitioner: Partitioner, serializer: Serializer = SparkEnv.get.serializer, keyOrdering: Option[Ordering[K]] = None, aggregator: Option[Aggregator[K, V, C]] = None, mapSideCombine: Boolean = false, shuffleWriterProcessor: ShuffleWriteProcessor = new ShuffleWriteProcessor, rowBasedChecksums: Array[RowBasedChecksum] = ShuffleDependency.EMPTY_ROW_BASED_CHECKSUMS, checksumMismatchFullRetryEnabled: Boolean = false)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[C])
- _rdd
the parent RDD
- partitioner
partitioner used to partition the shuffle output
- serializer
Serializer to use. If not set explicitly then the default serializer, as specified by
spark.serializerconfig option, will be used.- keyOrdering
key ordering for RDD's shuffles
- aggregator
map/reduce-side aggregator for RDD's shuffle
- mapSideCombine
whether to perform partial aggregation (also known as map-side combine)
- shuffleWriterProcessor
the processor to control the write behavior in ShuffleMapTask
- rowBasedChecksums
the row-based checksums for each shuffle partition
Type Members
- implicit class LogStringContext extends AnyRef
- Definition Classes
- Logging
Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- def MDC(key: LogKey, value: Any): MDC
- Attributes
- protected
- Definition Classes
- Logging
- val aggregator: Option[Aggregator[K, V, C]]
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- val checksumMismatchFullRetryEnabled: Boolean
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def equals(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef → Any
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- def getMergerLocs: Seq[BlockManagerId]
- def hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- def initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- def isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- val keyOrdering: Option[Ordering[K]]
- def log: Logger
- Attributes
- protected
- Definition Classes
- Logging
- def logBasedOnLevel(level: Level)(f: => MessageWithContext): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logName: String
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- val mapSideCombine: Boolean
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def newShuffleMergeState(): Unit
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- val partitioner: Partitioner
- def rdd: RDD[Product2[K, V]]
- Definition Classes
- ShuffleDependency → Dependency
- val rowBasedChecksums: Array[RowBasedChecksum]
- val serializer: Serializer
- def setMergerLocs(mergerLocs: Seq[BlockManagerId]): Unit
- val shuffleHandle: ShuffleHandle
- val shuffleId: Int
- def shuffleMergeAllowed: Boolean
- def shuffleMergeEnabled: Boolean
- def shuffleMergeFinalized: Boolean
Returns true if push-based shuffle is disabled or if the shuffle merge for this shuffle is finalized.
- def shuffleMergeId: Int
- val shuffleWriterProcessor: ShuffleWriteProcessor
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toString(): String
- Definition Classes
- AnyRef → Any
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- def withLogContext(context: Map[String, String])(body: => Unit): Unit
- Attributes
- protected
- Definition Classes
- Logging
Deprecated Value Members
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable]) @Deprecated
- Deprecated
(Since version 9)