InsertIntoHiveTable

Instance Constructors

new InsertIntoHiveTable(table: MetastoreRelation, partition: Map[String, Option[String]], child: SparkPlan, overwrite: Boolean)(sc: HiveContext)

Type Members

implicit class typeInfoConversions extends AnyRef

Definition Classes
HiveInspectors

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
def apply(number: Int): SparkPlan

Definition Classes
TreeNode
def argString: String

Definition Classes
TreeNode
def asCode: String

Definition Classes
TreeNode
final def asInstanceOf[T0]: T0

Definition Classes
Any
val child: SparkPlan

Definition Classes
InsertIntoHiveTable → UnaryNode
def children: List[SparkPlan]

Definition Classes
UnaryNode
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val codegenEnabled: Boolean

Definition Classes
SparkPlan
def collect[B](pf: PartialFunction[SparkPlan, B]): Seq[B]

Definition Classes
TreeNode
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def execute(): RDD[catalyst.expressions.Row]

Definition Classes
Command
def executeCollect(): Array[catalyst.expressions.Row]

Definition Classes
Command
def expressions: Seq[Expression]

Definition Classes
QueryPlan
def fastEquals(other: TreeNode[_]): Boolean

Definition Classes
TreeNode
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def flatMap[A](f: (SparkPlan) ⇒ TraversableOnce[A]): Seq[A]

Definition Classes
TreeNode
def foreach(f: (SparkPlan) ⇒ Unit): Unit

Definition Classes
TreeNode
def generateTreeString(depth: Int, builder: StringBuilder): StringBuilder

Attributes
protected
Definition Classes
TreeNode
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getNodeNumbered(number: MutableInt): SparkPlan

Attributes
protected
Definition Classes
TreeNode
def inputSet: AttributeSet

Definition Classes
QueryPlan
def inspectorToDataType(inspector: ObjectInspector): catalyst.types.DataType

Definition Classes
HiveInspectors
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def javaClassToDataType(clz: Class[_]): catalyst.types.DataType

Definition Classes
HiveInspectors
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def makeCopy(newArgs: Array[AnyRef]): InsertIntoHiveTable.this.type

Overridden make copy also propogates sqlContext to copied plan.
Overridden make copy also propogates sqlContext to copied plan.

Definition Classes
SparkPlan → TreeNode
def map[A](f: (SparkPlan) ⇒ A): Seq[A]

Definition Classes
TreeNode
def mapChildren(f: (SparkPlan) ⇒ SparkPlan): InsertIntoHiveTable.this.type

Definition Classes
TreeNode
def missingInput: AttributeSet

Definition Classes
QueryPlan
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def newMutableProjection(expressions: Seq[Expression], inputSchema: Seq[Attribute]): () ⇒ MutableProjection

Attributes
protected
Definition Classes
SparkPlan
def newOrdering(order: Seq[SortOrder], inputSchema: Seq[Attribute]): Ordering[catalyst.expressions.Row]

Attributes
protected
Definition Classes
SparkPlan
def newPredicate(expression: Expression, inputSchema: Seq[Attribute]): (catalyst.expressions.Row) ⇒ Boolean

Attributes
protected
Definition Classes
SparkPlan
def newProjection(expressions: Seq[Expression], inputSchema: Seq[Attribute]): Projection

Attributes
protected
Definition Classes
SparkPlan
def nodeName: String

Definition Classes
TreeNode
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def numberedTreeString: String

Definition Classes
TreeNode
def otherCopyArgs: List[HiveContext]

Definition Classes
InsertIntoHiveTable → TreeNode
def output: Seq[Attribute]

Definition Classes
InsertIntoHiveTable → QueryPlan
lazy val outputClass: Class[_ <: Writable]
def outputPartitioning: Partitioning

Specifies how data is partitioned across different nodes in the cluster.
Specifies how data is partitioned across different nodes in the cluster.

Definition Classes
UnaryNode → SparkPlan
def outputSet: AttributeSet

Definition Classes
QueryPlan
val overwrite: Boolean
val partition: Map[String, Option[String]]
def printSchema(): Unit

Definition Classes
QueryPlan
def references: AttributeSet

Definition Classes
QueryPlan
def requiredChildDistribution: Seq[Distribution]

Specifies any partition requirements on the input data for this operator.
Specifies any partition requirements on the input data for this operator.

Definition Classes
SparkPlan
def saveAsHiveFile(rdd: RDD[catalyst.expressions.Row], valueClass: Class[_], fileSinkConf: ShimFileSinkDesc, conf: SerializableWritable[JobConf], writerContainer: SparkHiveWriterContainer): Unit
def schema: catalyst.types.StructType

Definition Classes
QueryPlan
def schemaString: String

Definition Classes
QueryPlan
lazy val sideEffectResult: Seq[catalyst.expressions.Row]

Inserts all the rows in the table into Hive.
Inserts all the rows in the table into Hive. Row objects are properly serialized with the org.apache.hadoop.hive.serde2.SerDe and the org.apache.hadoop.mapred.OutputFormat provided by the table definition.
Note: this is run once and then kept to avoid double insertions.

Attributes
protected[org.apache.spark.sql]
Definition Classes
InsertIntoHiveTable → Command
def simpleString: String

Definition Classes
QueryPlan → TreeNode
def sparkContext: SparkContext

Attributes
protected
Definition Classes
SparkPlan
val sqlContext: SQLContext

A handle to the SQL Context that was used to create this plan.
A handle to the SQL Context that was used to create this plan. Since many operators need access to the sqlContext for RDD operations or configuration this field is automatically populated by the query planning infrastructure.

Attributes
protected[org.apache.spark]
Definition Classes
SparkPlan
def statePrefix: String

Attributes
protected
Definition Classes
QueryPlan
def stringArgs: Iterator[Any]

Attributes
protected
Definition Classes
TreeNode
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val table: MetastoreRelation
def toInspector(expr: Expression): ObjectInspector

Definition Classes
HiveInspectors
def toInspector(dataType: catalyst.types.DataType): ObjectInspector

Definition Classes
HiveInspectors
def toString(): String

Definition Classes
TreeNode → AnyRef → Any
def transform(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan

Definition Classes
TreeNode
def transformAllExpressions(rule: PartialFunction[Expression, Expression]): InsertIntoHiveTable.this.type

Definition Classes
QueryPlan
def transformChildrenDown(rule: PartialFunction[SparkPlan, SparkPlan]): InsertIntoHiveTable.this.type

Definition Classes
TreeNode
def transformChildrenUp(rule: PartialFunction[SparkPlan, SparkPlan]): InsertIntoHiveTable.this.type

Definition Classes
TreeNode
def transformDown(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan

Definition Classes
TreeNode
def transformExpressions(rule: PartialFunction[Expression, Expression]): InsertIntoHiveTable.this.type

Definition Classes
QueryPlan
def transformExpressionsDown(rule: PartialFunction[Expression, Expression]): InsertIntoHiveTable.this.type

Definition Classes
QueryPlan
def transformExpressionsUp(rule: PartialFunction[Expression, Expression]): InsertIntoHiveTable.this.type

Definition Classes
QueryPlan
def transformUp(rule: PartialFunction[SparkPlan, SparkPlan]): SparkPlan

Definition Classes
TreeNode
def treeString: String

Definition Classes
TreeNode
def unwrap(data: Any, oi: ObjectInspector): Any

Converts hive types to native catalyst types.
Converts hive types to native catalyst types.
data
the data in Hive type
oi
the ObjectInspector associated with the Hive Type
returns
convert the data into catalyst type TODO return the function of (data => Any) instead for performance consideration

Definition Classes
HiveInspectors
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withNewChildren(newChildren: Seq[SparkPlan]): InsertIntoHiveTable.this.type

Definition Classes
TreeNode
def wrap(row: Seq[Any], inspectors: Seq[ObjectInspector], cache: Array[AnyRef]): Array[AnyRef]

Definition Classes
HiveInspectors
def wrap(a: Any, oi: ObjectInspector): AnyRef

Converts native catalyst types to the types expected by Hive
Converts native catalyst types to the types expected by Hive
a
the value to be wrapped
oi
This ObjectInspector associated with the value returned by this function, and the ObjectInspector should also be consistent with those returned from toInspector: DataType => ObjectInspector and toInspector: Expression => ObjectInspector

Definition Classes
HiveInspectors
def wrapperFor(oi: ObjectInspector): (Any) ⇒ Any

Wraps with Hive types based on object inspector.
Wraps with Hive types based on object inspector. TODO: Consolidate all hive OI/data interface code.

Attributes
protected
Definition Classes
HiveInspectors

case class InsertIntoHiveTable(table: MetastoreRelation, partition: Map[String, Option[String]], child: SparkPlan, overwrite: Boolean)(sc: HiveContext) extends SparkPlan with UnaryNode with Command with HiveInspectors with Product with Serializable

Instance Constructors

new InsertIntoHiveTable(table: MetastoreRelation, partition: Map[String, Option[String]], child: SparkPlan, overwrite: Boolean)(sc: HiveContext)

Type Members

implicit class typeInfoConversions extends AnyRef

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def apply(number: Int): SparkPlan

def argString: String

def asCode: String

final def asInstanceOf[T0]: T0

val child: SparkPlan

def children: List[SparkPlan]

def clone(): AnyRef

val codegenEnabled: Boolean

def collect[B](pf: PartialFunction[SparkPlan, B]): Seq[B]

final def eq(arg0: AnyRef): Boolean

def execute(): RDD[catalyst.expressions.Row]

def executeCollect(): Array[catalyst.expressions.Row]

def expressions: Seq[Expression]

def fastEquals(other: TreeNode[_]): Boolean

def finalize(): Unit

def flatMap[A](f: (SparkPlan) ⇒ TraversableOnce[A]): Seq[A]

def foreach(f: (SparkPlan) ⇒ Unit): Unit

def generateTreeString(depth: Int, builder: StringBuilder): StringBuilder

final def getClass(): Class[_]

def getNodeNumbered(number: MutableInt): SparkPlan

def inputSet: AttributeSet

def inspectorToDataType(inspector: ObjectInspector): catalyst.types.DataType

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def javaClassToDataType(clz: Class[_]): catalyst.types.DataType

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def makeCopy(newArgs: Array[AnyRef]): InsertIntoHiveTable.this.type

def map[A](f: (SparkPlan) ⇒ A): Seq[A]

def mapChildren(f: (SparkPlan) ⇒ SparkPlan): InsertIntoHiveTable.this.type

def missingInput: AttributeSet

final def ne(arg0: AnyRef): Boolean

def newMutableProjection(expressions: Seq[Expression], inputSchema: Seq[Attribute]): () ⇒ MutableProjection

def newOrdering(order: Seq[SortOrder], inputSchema: Seq[Attribute]): Ordering[catalyst.expressions.Row]

def newPredicate(expression: Expression, inputSchema: Seq[Attribute]): (catalyst.expressions.Row) ⇒ Boolean

def newProjection(expressions: Seq[Expression], inputSchema: Seq[Attribute]): Projection

def nodeName: String

final def notify(): Unit

final def notifyAll(): Unit

def numberedTreeString: String

def otherCopyArgs: List[HiveContext]

def output: Seq[Attribute]

lazy val outputClass: Class[_ <: Writable]

def outputPartitioning: Partitioning

def outputSet: AttributeSet

val overwrite: Boolean

val partition: Map[String, Option[String]]

def printSchema(): Unit

def references: AttributeSet

def requiredChildDistribution: Seq[Distribution]

def saveAsHiveFile(rdd: RDD[catalyst.expressions.Row], valueClass: Class[_], fileSinkConf: ShimFileSinkDesc, conf: SerializableWritable[JobConf], writerContainer: SparkHiveWriterContainer): Unit

def schema: catalyst.types.StructType

def schemaString: String

lazy val sideEffectResult: Seq[catalyst.expressions.Row]

def simpleString: String

def sparkContext: SparkContext

val sqlContext: SQLContext

def statePrefix: String