TestSQLContext

Type Members

implicit class DslAttribute extends AnyRef

Definition Classes
ExpressionConversions
implicit class DslExpression extends ImplicitOperators

Definition Classes
ExpressionConversions
implicit class DslString extends ImplicitOperators

Definition Classes
ExpressionConversions
implicit class DslSymbol extends ImplicitAttribute

Definition Classes
ExpressionConversions
abstract class ImplicitAttribute extends ImplicitOperators

Definition Classes
ExpressionConversions
abstract class QueryExecution extends AnyRef

The primary workflow for executing relational queries using Spark.
class SparkPlanner extends SparkStrategies

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
lazy val analyzer: Analyzer

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
final def asInstanceOf[T0]: T0

Definition Classes
Any
implicit def binaryToLiteral(a: Array[Byte]): Literal

Definition Classes
ExpressionConversions
implicit def booleanToLiteral(b: Boolean): Literal

Definition Classes
ExpressionConversions
implicit def byteToLiteral(b: Byte): Literal

Definition Classes
ExpressionConversions
def cacheTable(tableName: String): Unit

Caches the specified table in-memory.
Caches the specified table in-memory.

Definition Classes
SQLContext
lazy val catalog: Catalog

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createParquetFile[A <: Product](path: String, allowExisting: Boolean = true, conf: Configuration = new Configuration())(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

:: Experimental :: Creates an empty parquet file with the schema of class A, which can be registered as a table.
:: Experimental :: Creates an empty parquet file with the schema of class A, which can be registered as a table. This registered table can be used as the target of future insertInto operations.
```
val sqlContext = new SQLContext(...)
import sqlContext._

case class Person(name: String, age: Int)
createParquetFile[Person]("path/to/file.parquet").registerAsTable("people")
sql("INSERT INTO people SELECT 'michael', 29")
```
A
A case class type that describes the desired schema of the parquet file to be created.
path
The path where the directory containing parquet metadata should be created. Data inserted into this table will also be stored at this location.
allowExisting
When false, an exception will be thrown if this directory already exists.
conf
A Hadoop configuration object that can be used to specify options to the parquet output format.

Definition Classes
SQLContext
Annotations
@Experimental()
implicit def createSchemaRDD[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

Creates a SchemaRDD from an RDD of case classes.
Creates a SchemaRDD from an RDD of case classes.

Definition Classes
SQLContext
implicit def decimalToLiteral(d: BigDecimal): Literal

Definition Classes
ExpressionConversions
implicit def doubleToLiteral(d: Double): Literal

Definition Classes
ExpressionConversions
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def executePlan(plan: LogicalPlan): QueryExecution

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def executeSql(sql: String): QueryExecution

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
implicit def floatToLiteral(f: Float): Literal

Definition Classes
ExpressionConversions
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
implicit def intToLiteral(i: Int): Literal

Definition Classes
ExpressionConversions
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
lazy val logger: Logger

Attributes
protected
Definition Classes
Logging
implicit def logicalPlanToSparkQuery(plan: LogicalPlan): SchemaRDD

:: DeveloperApi :: Allows catalyst LogicalPlans to be executed as a SchemaRDD.
:: DeveloperApi :: Allows catalyst LogicalPlans to be executed as a SchemaRDD. Note that the LogicalPlan interface is considered internal, and thus not guaranteed to be stable. As a result, using them directly is not recommended.

Definition Classes
SQLContext
Annotations
@DeveloperApi()
implicit def longToLiteral(l: Long): Literal

Definition Classes
ExpressionConversions
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val optimizer: Optimizer.type

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def parquetFile(path: String): SchemaRDD

Loads a Parquet file, returning the result as a SchemaRDD.
Loads a Parquet file, returning the result as a SchemaRDD.

Definition Classes
SQLContext
def parseSql(sql: String): LogicalPlan

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
val parser: SqlParser

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
val planner: SparkPlanner

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
val prepareForExecution: RuleExecutor[SparkPlan] { val batches: List[this.Batch] }

Prepares a planned SparkPlan for execution by binding references to specific ordinals, and inserting shuffle operations as needed.
Prepares a planned SparkPlan for execution by binding references to specific ordinals, and inserting shuffle operations as needed.

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def registerRDDAsTable(rdd: SchemaRDD, tableName: String): Unit

Registers the given RDD as a temporary table in the catalog.
Registers the given RDD as a temporary table in the catalog. Temporary tables exist only during the lifetime of this instance of SQLContext.

Definition Classes
SQLContext
implicit def shortToLiteral(s: Short): Literal

Definition Classes
ExpressionConversions
val sparkContext: SparkContext

Definition Classes
SQLContext
def sql(sqlText: String): SchemaRDD

Executes a SQL query using Spark, returning the result as a SchemaRDD.
Executes a SQL query using Spark, returning the result as a SchemaRDD.

Definition Classes
SQLContext
implicit def stringToLiteral(s: String): Literal

Definition Classes
ExpressionConversions
implicit def symbolToUnresolvedAttribute(s: Symbol): UnresolvedAttribute

Definition Classes
ExpressionConversions
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def table(tableName: String): SchemaRDD

Returns the specified table as a SchemaRDD
Returns the specified table as a SchemaRDD

Definition Classes
SQLContext
implicit def timestampToLiteral(t: Timestamp): Literal

Definition Classes
ExpressionConversions
def toString(): String

Definition Classes
AnyRef → Any
def uncacheTable(tableName: String): Unit

Removes the specified table from the in-memory cache.
Removes the specified table from the in-memory cache.

Definition Classes
SQLContext
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

object TestSQLContext extends SQLContext

Type Members

implicit class DslAttribute extends AnyRef

implicit class DslExpression extends ImplicitOperators

implicit class DslString extends ImplicitOperators

implicit class DslSymbol extends ImplicitAttribute

abstract class ImplicitAttribute extends ImplicitOperators

abstract class QueryExecution extends AnyRef

class SparkPlanner extends SparkStrategies

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

lazy val analyzer: Analyzer

final def asInstanceOf[T0]: T0

implicit def binaryToLiteral(a: Array[Byte]): Literal

implicit def booleanToLiteral(b: Boolean): Literal

implicit def byteToLiteral(b: Byte): Literal

def cacheTable(tableName: String): Unit

lazy val catalog: Catalog

def clone(): AnyRef

def createParquetFile[A <: Product](path: String, allowExisting: Boolean = true, conf: Configuration = new Configuration())(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

implicit def createSchemaRDD[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

implicit def decimalToLiteral(d: BigDecimal): Literal

implicit def doubleToLiteral(d: Double): Literal

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executePlan(plan: LogicalPlan): QueryExecution

def executeSql(sql: String): QueryExecution

def finalize(): Unit

implicit def floatToLiteral(f: Float): Literal

final def getClass(): Class[_]

def hashCode(): Int

implicit def intToLiteral(i: Int): Literal

final def isInstanceOf[T0]: Boolean

lazy val logger: Logger

implicit def logicalPlanToSparkQuery(plan: LogicalPlan): SchemaRDD

implicit def longToLiteral(l: Long): Literal

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val optimizer: Optimizer.type

def parquetFile(path: String): SchemaRDD

def parseSql(sql: String): LogicalPlan

val parser: SqlParser

val planner: SparkPlanner

val prepareForExecution: RuleExecutor[SparkPlan] { val batches: List[this.Batch] }

def registerRDDAsTable(rdd: SchemaRDD, tableName: String): Unit

implicit def shortToLiteral(s: Short): Literal

val sparkContext: SparkContext

def sql(sqlText: String): SchemaRDD

implicit def stringToLiteral(s: String): Literal

implicit def symbolToUnresolvedAttribute(s: Symbol): UnresolvedAttribute

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): SchemaRDD

implicit def timestampToLiteral(t: Timestamp): Literal

def toString(): String

def uncacheTable(tableName: String): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from SQLContext

Inherited from Serializable

Inherited from Serializable

Inherited from ExpressionConversions

Inherited from com.typesafe.scalalogging.slf4j.Logging

Inherited from AnyRef

Inherited from Any

Spark SQL Functions

Ungrouped