org.apache.spark.sql

ColumnName

class ColumnName extends Column

:: Experimental :: A convenient class used for constructing schema.

Annotations
@Experimental()
Linear Supertypes
Column, AnyRef, Any
Ordering
  1. Grouped
  2. Alphabetic
  3. By inheritance
Inherited
  1. ColumnName
  2. Column
  3. AnyRef
  4. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Instance Constructors

  1. new ColumnName(name: String)

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any

  3. def !==(other: Any): Column

    Inequality test.

    Inequality test.

    // Scala:
df.select( df("colA") !== df("colB") )
df.select( !(df("colA") === df("colB")) )

// Java:
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").notEqual(col("colB")) );
    Definition Classes
    Column

  4. final def ##(): Int

    Definition Classes
    AnyRef → Any

  5. def %(other: Any): Column

    Modulo (a.

    Modulo (a.k.a. remainder) expression.

    Definition Classes
    Column

  6. def &&(other: Any): Column

    Boolean AND.

    Boolean AND.

    // Scala: The following selects people that are in school and employed at the same time.
people.select( people("inSchool") && people("isEmployed") )

// Java:
people.select( people("inSchool").and(people("isEmployed")) );
    Definition Classes
    Column

  7. def *(other: Any): Column

    Multiplication of this expression and another expression.

    Multiplication of this expression and another expression.

    // Scala: The following multiplies a person's height by their weight.
people.select( people("height") * people("weight") )

// Java:
people.select( people("height").multiply(people("weight")) );
    Definition Classes
    Column

  8. def +(other: Any): Column

    Sum of this expression and another expression.

    Sum of this expression and another expression.

    // Scala: The following selects the sum of a person's height and weight.
people.select( people("height") + people("weight") )

// Java:
people.select( people("height").plus(people("weight")) );
    Definition Classes
    Column

  9. def -(other: Any): Column

    Subtraction.

    Subtraction. Subtract the other expression from this expression.

    // Scala: The following selects the difference between people's height and their weight.
people.select( people("height") - people("weight") )

// Java:
people.select( people("height").minus(people("weight")) );
    Definition Classes
    Column

  10. def /(other: Any): Column

    Division this expression by another expression.

    Division this expression by another expression.

    // Scala: The following divides a person's height by their weight.
people.select( people("height") / people("weight") )

// Java:
people.select( people("height").divide(people("weight")) );
    Definition Classes
    Column

  11. def <(other: Any): Column

    Less than.

    Less than.

    // Scala: The following selects people younger than 21.
people.select( people("age") < 21 )

// Java:
people.select( people("age").lt(21) );
    Definition Classes
    Column

  12. def <=(other: Any): Column

    Less than or equal to.

    Less than or equal to.

    // Scala: The following selects people age 21 or younger than 21.
people.select( people("age") <= 21 )

// Java:
people.select( people("age").leq(21) );
    Definition Classes
    Column

  13. def <=>(other: Any): Column

    Equality test that is safe for null values.

    Equality test that is safe for null values.

    Definition Classes
    Column

  14. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  15. final def ==(arg0: Any): Boolean

    Definition Classes
    Any

  16. def ===(other: Any): Column

    Equality test.

    Equality test.

    // Scala:
df.filter( df("colA") === df("colB") )

// Java
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").equalTo(col("colB")) );
    Definition Classes
    Column

  17. def >(other: Any): Column

    Greater than.

    Greater than.

    // Scala: The following selects people older than 21.
people.select( people("age") > 21 )

// Java:
import static org.apache.spark.sql.functions.*;
people.select( people("age").gt(21) );
    Definition Classes
    Column

  18. def >=(other: Any): Column

    Greater than or equal to an expression.

    Greater than or equal to an expression.

    // Scala: The following selects people age 21 or older than 21.
people.select( people("age") >= 21 )

// Java:
people.select( people("age").geq(21) )
    Definition Classes
    Column

  19. def and(other: Column): Column

    Boolean AND.

    Boolean AND.

    // Scala: The following selects people that are in school and employed at the same time.
people.select( people("inSchool") && people("isEmployed") )

// Java:
people.select( people("inSchool").and(people("isEmployed")) );
    Definition Classes
    Column

  20. def array(dataType: DataType): StructField

    Creates a new AttributeReference of type array

  21. def as(alias: Symbol): Column

    Gives the column an alias.

    Gives the column an alias.

    // Renames colA to colB in select output.
df.select($"colA".as('colB))
    Definition Classes
    Column

  22. def as(alias: String): Column

    Gives the column an alias.

    Gives the column an alias.

    // Renames colA to colB in select output.
df.select($"colA".as("colB"))
    Definition Classes
    Column

  23. final def asInstanceOf[T0]: T0

    Definition Classes
    Any

  24. def asc: Column

    Returns an ordering used in sorting.

    Returns an ordering used in sorting.

    // Scala: sort a DataFrame by age column in ascending order.
df.sort(df("age").asc)

// Java
df.sort(df.col("age").asc());
    Definition Classes
    Column

  25. def binary: StructField

    Creates a new AttributeReference of type binary

  26. def boolean: StructField

    Creates a new AttributeReference of type boolean

  27. def byte: StructField

    Creates a new AttributeReference of type byte

  28. def cast(to: String): Column

    Casts the column to a different data type, using the canonical string representation of the type.

    Casts the column to a different data type, using the canonical string representation of the type. The supported types are: string, boolean, byte, short, int, long, float, double, decimal, date, timestamp.

    // Casts colA to integer.
df.select(df("colA").cast("int"))
    Definition Classes
    Column

  29. def cast(to: DataType): Column

    Casts the column to a different data type.

    Casts the column to a different data type.

    // Casts colA to IntegerType.
import org.apache.spark.sql.types.IntegerType
df.select(df("colA").cast(IntegerType))

// equivalent to
df.select(df("colA").cast("int"))
    Definition Classes
    Column

  30. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  31. def contains(other: Any): Column

    Contains the other element.

    Contains the other element.

    Definition Classes
    Column

  32. def date: StructField

    Creates a new AttributeReference of type date

  33. def decimal(precision: Int, scale: Int): StructField

    Creates a new AttributeReference of type decimal

  34. def decimal: StructField

    Creates a new AttributeReference of type decimal

  35. def desc: Column

    Returns an ordering used in sorting.

    Returns an ordering used in sorting.

    // Scala: sort a DataFrame by age column in descending order.
df.sort(df("age").desc)

// Java
df.sort(df.col("age").desc());
    Definition Classes
    Column

  36. def divide(other: Any): Column

    Division this expression by another expression.

    Division this expression by another expression.

    // Scala: The following divides a person's height by their weight.
people.select( people("height") / people("weight") )

// Java:
people.select( people("height").divide(people("weight")) );
    Definition Classes
    Column

  37. def double: StructField

    Creates a new AttributeReference of type double

  38. def endsWith(literal: String): Column

    String ends with another string literal.

    String ends with another string literal.

    Definition Classes
    Column

  39. def endsWith(other: Column): Column

    String ends with.

    String ends with.

    Definition Classes
    Column

  40. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  41. def eqNullSafe(other: Any): Column

    Equality test that is safe for null values.

    Equality test that is safe for null values.

    Definition Classes
    Column

  42. def equalTo(other: Any): Column

    Equality test.

    Equality test.

    // Scala:
df.filter( df("colA") === df("colB") )

// Java
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").equalTo(col("colB")) );
    Definition Classes
    Column

  43. def equals(that: Any): Boolean

    Definition Classes
    Column → AnyRef → Any

  44. def explain(extended: Boolean): Unit

    Prints the expression to the console for debugging purpose.

    Prints the expression to the console for debugging purpose.

    Definition Classes
    Column

  45. val expr: Expression

    Attributes
    protected[org.apache.spark.sql]
    Definition Classes
    Column

  46. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  47. def float: StructField

    Creates a new AttributeReference of type float

  48. def geq(other: Any): Column

    Greater than or equal to an expression.

    Greater than or equal to an expression.

    // Scala: The following selects people age 21 or older than 21.
people.select( people("age") >= 21 )

// Java:
people.select( people("age").geq(21) )
    Definition Classes
    Column

  49. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any

  50. def getField(fieldName: String): Column

    An expression that gets a field by name in a StructField.

    An expression that gets a field by name in a StructField.

    Definition Classes
    Column

  51. def getItem(ordinal: Int): Column

    An expression that gets an item at position ordinal out of an array.

    An expression that gets an item at position ordinal out of an array.

    Definition Classes
    Column

  52. def gt(other: Any): Column

    Greater than.

    Greater than.

    // Scala: The following selects people older than 21.
people.select( people("age") > lit(21) )

// Java:
import static org.apache.spark.sql.functions.*;
people.select( people("age").gt(21) );
    Definition Classes
    Column

  53. def hashCode(): Int

    Definition Classes
    Column → AnyRef → Any

  54. def in(list: Column*): Column

    A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

    A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

    Definition Classes
    Column
    Annotations
    @varargs()

  55. def int: StructField

    Creates a new AttributeReference of type int

  56. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any

  57. def isNotNull: Column

    True if the current expression is NOT null.

    True if the current expression is NOT null.

    Definition Classes
    Column

  58. def isNull: Column

    True if the current expression is null.

    True if the current expression is null.

    Definition Classes
    Column

  59. def leq(other: Any): Column

    Less than or equal to.

    Less than or equal to.

    // Scala: The following selects people age 21 or younger than 21.
people.select( people("age") <= 21 )

// Java:
people.select( people("age").leq(21) );
    Definition Classes
    Column

  60. def like(literal: String): Column

    SQL like expression.

    SQL like expression.

    Definition Classes
    Column

  61. def long: StructField

    Creates a new AttributeReference of type long

  62. def lt(other: Any): Column

    Less than.

    Less than.

    // Scala: The following selects people younger than 21.
people.select( people("age") < 21 )

// Java:
people.select( people("age").lt(21) );
    Definition Classes
    Column

  63. def map(mapType: MapType): StructField

  64. def map(keyType: DataType, valueType: DataType): StructField

    Creates a new AttributeReference of type map

  65. def minus(other: Any): Column

    Subtraction.

    Subtraction. Subtract the other expression from this expression.

    // Scala: The following selects the difference between people's height and their weight.
people.select( people("height") - people("weight") )

// Java:
people.select( people("height").minus(people("weight")) );
    Definition Classes
    Column

  66. def mod(other: Any): Column

    Modulo (a.

    Modulo (a.k.a. remainder) expression.

    Definition Classes
    Column

  67. def multiply(other: Any): Column

    Multiplication of this expression and another expression.

    Multiplication of this expression and another expression.

    // Scala: The following multiplies a person's height by their weight.
people.select( people("height") * people("weight") )

// Java:
people.select( people("height").multiply(people("weight")) );
    Definition Classes
    Column

  68. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  69. def notEqual(other: Any): Column

    Inequality test.

    Inequality test.

    // Scala:
df.select( df("colA") !== df("colB") )
df.select( !(df("colA") === df("colB")) )

// Java:
import static org.apache.spark.sql.functions.*;
df.filter( col("colA").notEqual(col("colB")) );
    Definition Classes
    Column

  70. final def notify(): Unit

    Definition Classes
    AnyRef

  71. final def notifyAll(): Unit

    Definition Classes
    AnyRef

  72. def or(other: Column): Column

    Boolean OR.

    Boolean OR.

    // Scala: The following selects people that are in school or employed.
people.filter( people("inSchool") || people("isEmployed") )

// Java:
people.filter( people("inSchool").or(people("isEmployed")) );
    Definition Classes
    Column

  73. def plus(other: Any): Column

    Sum of this expression and another expression.

    Sum of this expression and another expression.

    // Scala: The following selects the sum of a person's height and weight.
people.select( people("height") + people("weight") )

// Java:
people.select( people("height").plus(people("weight")) );
    Definition Classes
    Column

  74. def rlike(literal: String): Column

    SQL RLIKE expression (LIKE with Regex).

    SQL RLIKE expression (LIKE with Regex).

    Definition Classes
    Column

  75. def short: StructField

    Creates a new AttributeReference of type short

  76. def startsWith(literal: String): Column

    String starts with another string literal.

    String starts with another string literal.

    Definition Classes
    Column

  77. def startsWith(other: Column): Column

    String starts with.

    String starts with.

    Definition Classes
    Column

  78. def string: StructField

    Creates a new AttributeReference of type string

  79. def struct(structType: StructType): StructField

  80. def struct(fields: StructField*): StructField

    Creates a new AttributeReference of type struct

  81. def substr(startPos: Int, len: Int): Column

    An expression that returns a substring.

    An expression that returns a substring.

    startPos

    starting position.

    len

    length of the substring.

    Definition Classes
    Column

  82. def substr(startPos: Column, len: Column): Column

    An expression that returns a substring.

    An expression that returns a substring.

    startPos

    expression for the starting position.

    len

    expression for the length of the substring.

    Definition Classes
    Column

  83. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef

  84. def timestamp: StructField

    Creates a new AttributeReference of type timestamp

  85. def toString(): String

    Definition Classes
    Column → AnyRef → Any

  86. def unary_!: Column

    Inversion of boolean expression, i.

    Inversion of boolean expression, i.e. NOT. {{ // Scala: select rows that are not active (isActive === false) df.filter( !df("isActive") )

    // Java: import static org.apache.spark.sql.functions.*; df.filter( not(df.col("isActive")) ); }}

    Definition Classes
    Column

  87. def unary_-: Column

    Unary minus, i.

    Unary minus, i.e. negate the expression.

    // Scala: select the amount column and negates all values.
df.select( -df("amount") )

// Java:
import static org.apache.spark.sql.functions.*;
df.select( negate(col("amount") );
    Definition Classes
    Column

  88. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  89. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  90. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  91. def ||(other: Any): Column

    Boolean OR.

    Boolean OR.

    // Scala: The following selects people that are in school or employed.
people.filter( people("inSchool") || people("isEmployed") )

// Java:
people.filter( people("inSchool").or(people("isEmployed")) );
    Definition Classes
    Column

Inherited from Column

Inherited from AnyRef

Inherited from Any

df_ops

expr_ops

java_expr_ops

Ungrouped