public interface SparkSessionExtensionsProvider extends scala.Function1<SparkSessionExtensions,scala.runtime.BoxedUnit>
Base trait for implementations used by SparkSessionExtensions
For example, now we have an external function named Age
to register as an extension for SparkSession:
package org.apache.spark.examples.extensions
import org.apache.spark.sql.catalyst.expressions.{CurrentDate, Expression, RuntimeReplaceable, SubtractDates}
case class Age(birthday: Expression, child: Expression) extends RuntimeReplaceable {
def this(birthday: Expression) = this(birthday, SubtractDates(CurrentDate(), birthday))
override def exprsReplaced: Seq[Expression] = Seq(birthday)
override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild)
}
We need to create our extension which inherits SparkSessionExtensionsProvider
Example:
package org.apache.spark.examples.extensions
import org.apache.spark.sql.{SparkSessionExtensions, SparkSessionExtensionsProvider}
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}
class MyExtensions extends SparkSessionExtensionsProvider {
override def apply(v1: SparkSessionExtensions): Unit = {
v1.injectFunction(
(new FunctionIdentifier("age"),
new ExpressionInfo(classOf[Age].getName, "age"),
(children: Seq[Expression]) => new Age(children.head)))
}
}
Then, we can inject MyExtensions
in three ways,
SparkSession.Builder
ServiceLoader
- Add to src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProviderSparkSessionExtensions
,
SparkSession.Builder
,
$init$, andThen, apply, apply$mcDD$sp, apply$mcDF$sp, apply$mcDI$sp, apply$mcDJ$sp, apply$mcFD$sp, apply$mcFF$sp, apply$mcFI$sp, apply$mcFJ$sp, apply$mcID$sp, apply$mcIF$sp, apply$mcII$sp, apply$mcIJ$sp, apply$mcJD$sp, apply$mcJF$sp, apply$mcJI$sp, apply$mcJJ$sp, apply$mcVD$sp, apply$mcVF$sp, apply$mcVI$sp, apply$mcVJ$sp, apply$mcZD$sp, apply$mcZF$sp, apply$mcZI$sp, apply$mcZJ$sp, compose, toString