Object

org.apache.spark.sql.SparkSession

implicits

Related Doc: package SparkSession

Permalink

object implicits extends SQLImplicits with Serializable

:: Experimental :: (Scala-specific) Implicit methods available in Scala for converting common Scala objects into DataFrames.

val sparkSession = SparkSession.builder.getOrCreate()
import sparkSession.implicits._
Annotations
@Experimental() @Evolving()
Source
SparkSession.scala
Since

2.0.0

Linear Supertypes
Serializable, Serializable, SQLImplicits, LowPrioritySQLImplicits, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. implicits
  2. Serializable
  3. Serializable
  4. SQLImplicits
  5. LowPrioritySQLImplicits
  6. AnyRef
  7. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. implicit class StringToColumn extends AnyRef

    Permalink

    Converts $"col name" into a Column.

    Converts $"col name" into a Column.

    Definition Classes
    SQLImplicits
    Since

    2.0.0

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  4. def _sqlContext: SQLContext

    Permalink
    Attributes
    protected
    Definition Classes
    implicitsSQLImplicits
  5. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any
  6. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  7. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  8. def equals(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  9. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  10. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any
  11. def hashCode(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  12. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any
  13. implicit def localSeqToDatasetHolder[T](s: Seq[T])(implicit arg0: Encoder[T]): DatasetHolder[T]

    Permalink

    Creates a Dataset from a local Seq.

    Creates a Dataset from a local Seq.

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  14. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  15. implicit def newBooleanArrayEncoder: Encoder[Array[Boolean]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  16. implicit def newBooleanEncoder: Encoder[Boolean]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  17. implicit def newBoxedBooleanEncoder: Encoder[Boolean]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  18. implicit def newBoxedByteEncoder: Encoder[Byte]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  19. implicit def newBoxedDoubleEncoder: Encoder[Double]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  20. implicit def newBoxedFloatEncoder: Encoder[Float]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  21. implicit def newBoxedIntEncoder: Encoder[Integer]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  22. implicit def newBoxedLongEncoder: Encoder[Long]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  23. implicit def newBoxedShortEncoder: Encoder[Short]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.0.0

  24. implicit def newByteArrayEncoder: Encoder[Array[Byte]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  25. implicit def newByteEncoder: Encoder[Byte]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  26. implicit def newDateEncoder: Encoder[Date]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.2.0

  27. implicit def newDoubleArrayEncoder: Encoder[Array[Double]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  28. implicit def newDoubleEncoder: Encoder[Double]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  29. implicit def newFloatArrayEncoder: Encoder[Array[Float]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  30. implicit def newFloatEncoder: Encoder[Float]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  31. implicit def newIntArrayEncoder: Encoder[Array[Int]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  32. implicit def newIntEncoder: Encoder[Int]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  33. implicit def newJavaDecimalEncoder: Encoder[BigDecimal]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.2.0

  34. implicit def newLongArrayEncoder: Encoder[Array[Long]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  35. implicit def newLongEncoder: Encoder[Long]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  36. implicit def newMapEncoder[T <: Map[_, _]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.3.0

  37. implicit def newProductArrayEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Array[A]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  38. implicit def newProductEncoder[T <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]

    Permalink

    Definition Classes
    LowPrioritySQLImplicits
    Since

    1.6.0

  39. implicit def newScalaDecimalEncoder: Encoder[BigDecimal]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.2.0

  40. implicit def newSequenceEncoder[T <: Seq[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.2.0

  41. implicit def newSetEncoder[T <: Set[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]

    Permalink

    Notice that we serialize Set to Catalyst array.

    Notice that we serialize Set to Catalyst array. The set property is only kept when manipulating the domain objects. The serialization format doesn't keep the set property. When we have a Catalyst array which contains duplicated elements and convert it to Dataset[Set[T]] by using the encoder, the elements will be de-duplicated.

    Definition Classes
    SQLImplicits
    Since

    2.3.0

  42. implicit def newShortArrayEncoder: Encoder[Array[Short]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  43. implicit def newShortEncoder: Encoder[Short]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  44. implicit def newStringArrayEncoder: Encoder[Array[String]]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.1

  45. implicit def newStringEncoder: Encoder[String]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  46. implicit def newTimeStampEncoder: Encoder[Timestamp]

    Permalink

    Definition Classes
    SQLImplicits
    Since

    2.2.0

  47. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef
  48. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef
  49. implicit def rddToDatasetHolder[T](rdd: RDD[T])(implicit arg0: Encoder[T]): DatasetHolder[T]

    Permalink

    Creates a Dataset from an RDD.

    Creates a Dataset from an RDD.

    Definition Classes
    SQLImplicits
    Since

    1.6.0

  50. implicit def symbolToColumn(s: Symbol): ColumnName

    Permalink

    An implicit conversion that turns a Scala Symbol into a Column.

    An implicit conversion that turns a Scala Symbol into a Column.

    Definition Classes
    SQLImplicits
    Since

    1.3.0

  51. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef
  52. def toString(): String

    Permalink
    Definition Classes
    AnyRef → Any
  53. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  54. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  55. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def newBooleanSeqEncoder: Encoder[Seq[Boolean]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  2. def newByteSeqEncoder: Encoder[Seq[Byte]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  3. def newDoubleSeqEncoder: Encoder[Seq[Double]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  4. def newFloatSeqEncoder: Encoder[Seq[Float]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  5. def newIntSeqEncoder: Encoder[Seq[Int]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  6. def newLongSeqEncoder: Encoder[Seq[Long]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  7. def newProductSeqEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Seq[A]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  8. def newShortSeqEncoder: Encoder[Seq[Short]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

  9. def newStringSeqEncoder: Encoder[Seq[String]]

    Permalink

    Definition Classes
    SQLImplicits
    Deprecated

    use newSequenceEncoder

    Since

    1.6.1

Inherited from Serializable

Inherited from Serializable

Inherited from SQLImplicits

Inherited from LowPrioritySQLImplicits

Inherited from AnyRef

Inherited from Any

Ungrouped