object implicits extends SQLImplicits with Serializable
(Scala-specific) Implicit methods available in Scala for converting
common Scala objects into DataFrame
s.
val sparkSession = SparkSession.builder.getOrCreate() import sparkSession.implicits._
- Source
- SparkSession.scala
- Since
2.0.0
- Alphabetic
- By Inheritance
- implicits
- Serializable
- SQLImplicits
- LowPrioritySQLImplicits
- AnyRef
- Any
- Hide All
- Show All
- Public
- Protected
Type Members
- implicit class StringToColumn extends AnyRef
Converts $"col name" into a Column.
Converts $"col name" into a Column.
- Definition Classes
- SQLImplicits
- Since
2.0.0
Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def equals(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef → Any
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- def hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- implicit def localSeqToDatasetHolder[T](s: Seq[T])(implicit arg0: Encoder[T]): DatasetHolder[T]
Creates a Dataset from a local Seq.
Creates a Dataset from a local Seq.
- Definition Classes
- SQLImplicits
- Since
1.6.0
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- implicit def newBooleanArrayEncoder: Encoder[Array[Boolean]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newBooleanEncoder: Encoder[Boolean]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newBoxedBooleanEncoder: Encoder[Boolean]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newBoxedByteEncoder: Encoder[Byte]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newBoxedDoubleEncoder: Encoder[Double]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newBoxedFloatEncoder: Encoder[Float]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newBoxedIntEncoder: Encoder[Integer]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newBoxedLongEncoder: Encoder[Long]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newBoxedShortEncoder: Encoder[Short]
- Definition Classes
- SQLImplicits
- Since
2.0.0
- implicit def newByteArrayEncoder: Encoder[Array[Byte]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newByteEncoder: Encoder[Byte]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newDateEncoder: Encoder[Date]
- Definition Classes
- SQLImplicits
- Since
2.2.0
- implicit def newDoubleArrayEncoder: Encoder[Array[Double]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newDoubleEncoder: Encoder[Double]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newDurationEncoder: Encoder[Duration]
- Definition Classes
- SQLImplicits
- Since
3.2.0
- implicit def newFloatArrayEncoder: Encoder[Array[Float]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newFloatEncoder: Encoder[Float]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newInstantEncoder: Encoder[Instant]
- Definition Classes
- SQLImplicits
- Since
3.0.0
- implicit def newIntArrayEncoder: Encoder[Array[Int]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newIntEncoder: Encoder[Int]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newJavaDecimalEncoder: Encoder[BigDecimal]
- Definition Classes
- SQLImplicits
- Since
2.2.0
- implicit def newJavaEnumEncoder[A <: Enum[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[A]
- Definition Classes
- SQLImplicits
- Since
3.2.0
- implicit def newLocalDateEncoder: Encoder[LocalDate]
- Definition Classes
- SQLImplicits
- Since
3.0.0
- implicit def newLocalDateTimeEncoder: Encoder[LocalDateTime]
- Definition Classes
- SQLImplicits
- Since
3.4.0
- implicit def newLongArrayEncoder: Encoder[Array[Long]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newLongEncoder: Encoder[Long]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newMapEncoder[T <: Map[_, _]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- SQLImplicits
- Since
2.3.0
- implicit def newPeriodEncoder: Encoder[Period]
- Definition Classes
- SQLImplicits
- Since
3.2.0
- implicit def newProductArrayEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Array[A]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newProductEncoder[T <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- LowPrioritySQLImplicits
- Since
1.6.0
- implicit def newScalaDecimalEncoder: Encoder[BigDecimal]
- Definition Classes
- SQLImplicits
- Since
2.2.0
- implicit def newSequenceEncoder[T <: Seq[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- SQLImplicits
- Since
2.2.0
- implicit def newSetEncoder[T <: Set[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
Notice that we serialize
Set
to Catalyst array.Notice that we serialize
Set
to Catalyst array. The set property is only kept when manipulating the domain objects. The serialization format doesn't keep the set property. When we have a Catalyst array which contains duplicated elements and convert it toDataset[Set[T]]
by using the encoder, the elements will be de-duplicated.- Definition Classes
- SQLImplicits
- Since
2.3.0
- implicit def newShortArrayEncoder: Encoder[Array[Short]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newShortEncoder: Encoder[Short]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newStringArrayEncoder: Encoder[Array[String]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
- implicit def newStringEncoder: Encoder[String]
- Definition Classes
- SQLImplicits
- Since
1.6.0
- implicit def newTimeStampEncoder: Encoder[Timestamp]
- Definition Classes
- SQLImplicits
- Since
2.2.0
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- implicit def rddToDatasetHolder[T](rdd: RDD[T])(implicit arg0: Encoder[T]): DatasetHolder[T]
Creates a Dataset from an RDD.
Creates a Dataset from an RDD.
- Definition Classes
- SQLImplicits
- Since
1.6.0
- def session: SparkSession
- Attributes
- protected
- Definition Classes
- implicits → SQLImplicits
- implicit def symbolToColumn(s: Symbol): ColumnName
An implicit conversion that turns a Scala
Symbol
into a Column.An implicit conversion that turns a Scala
Symbol
into a Column.- Definition Classes
- SQLImplicits
- Since
1.3.0
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toString(): String
- Definition Classes
- AnyRef → Any
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
Deprecated Value Members
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable]) @Deprecated
- Deprecated
(Since version 9)
- def newBooleanSeqEncoder: Encoder[Seq[Boolean]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newByteSeqEncoder: Encoder[Seq[Byte]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newDoubleSeqEncoder: Encoder[Seq[Double]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newFloatSeqEncoder: Encoder[Seq[Float]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newIntSeqEncoder: Encoder[Seq[Int]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newLongSeqEncoder: Encoder[Seq[Long]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newProductSeqEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Seq[A]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newShortSeqEncoder: Encoder[Seq[Short]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newStringSeqEncoder: Encoder[Seq[String]]
- Definition Classes
- SQLImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1