org.apache.spark.sql

Column

class Column extends AnyRef

:: Experimental :: A column in a DataFrame.

Annotations
@Experimental()
Linear Supertypes
AnyRef, Any
Known Subclasses
Ordering
  1. Grouped
  2. Alphabetic
  3. By inheritance
Inherited
  1. Column
  2. AnyRef
  3. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Instance Constructors

  1. new Column(name: String)

  2. new Column(expr: Expression)

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any
  3. def !==(other: Any): Column

    Inequality test.

    Inequality test.

    // Scala:
    df.select( df("colA") !== df("colB") )
    df.select( !(df("colA") === df("colB")) )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").notEqual(col("colB")) );
  4. final def ##(): Int

    Definition Classes
    AnyRef → Any
  5. def %(other: Any): Column

    Modulo (a.

    Modulo (a.k.a. remainder) expression.

  6. def &&(other: Any): Column

    Boolean AND.

    Boolean AND.

    // Scala: The following selects people that are in school and employed at the same time.
    people.select( people("inSchool") && people("isEmployed") )
    
    // Java:
    people.select( people("inSchool").and(people("isEmployed")) );
  7. def *(other: Any): Column

    Multiplication of this expression and another expression.

    Multiplication of this expression and another expression.

    // Scala: The following multiplies a person's height by their weight.
    people.select( people("height") * people("weight") )
    
    // Java:
    people.select( people("height").multiply(people("weight")) );
  8. def +(other: Any): Column

    Sum of this expression and another expression.

    Sum of this expression and another expression.

    // Scala: The following selects the sum of a person's height and weight.
    people.select( people("height") + people("weight") )
    
    // Java:
    people.select( people("height").plus(people("weight")) );
  9. def -(other: Any): Column

    Subtraction.

    Subtraction. Subtract the other expression from this expression.

    // Scala: The following selects the difference between people's height and their weight.
    people.select( people("height") - people("weight") )
    
    // Java:
    people.select( people("height").minus(people("weight")) );
  10. def /(other: Any): Column

    Division this expression by another expression.

    Division this expression by another expression.

    // Scala: The following divides a person's height by their weight.
    people.select( people("height") / people("weight") )
    
    // Java:
    people.select( people("height").divide(people("weight")) );
  11. def <(other: Any): Column

    Less than.

    Less than.

    // Scala: The following selects people younger than 21.
    people.select( people("age") < 21 )
    
    // Java:
    people.select( people("age").lt(21) );
  12. def <=(other: Any): Column

    Less than or equal to.

    Less than or equal to.

    // Scala: The following selects people age 21 or younger than 21.
    people.select( people("age") <= 21 )
    
    // Java:
    people.select( people("age").leq(21) );
  13. def <=>(other: Any): Column

    Equality test that is safe for null values.

  14. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  15. final def ==(arg0: Any): Boolean

    Definition Classes
    Any
  16. def ===(other: Any): Column

    Equality test.

    Equality test.

    // Scala:
    df.filter( df("colA") === df("colB") )
    
    // Java
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").equalTo(col("colB")) );
  17. def >(other: Any): Column

    Greater than.

    Greater than.

    // Scala: The following selects people older than 21.
    people.select( people("age") > 21 )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    people.select( people("age").gt(21) );
  18. def >=(other: Any): Column

    Greater than or equal to an expression.

    Greater than or equal to an expression.

    // Scala: The following selects people age 21 or older than 21.
    people.select( people("age") >= 21 )
    
    // Java:
    people.select( people("age").geq(21) )
  19. def and(other: Column): Column

    Boolean AND.

    Boolean AND.

    // Scala: The following selects people that are in school and employed at the same time.
    people.select( people("inSchool") && people("isEmployed") )
    
    // Java:
    people.select( people("inSchool").and(people("isEmployed")) );
  20. def as(alias: Symbol): Column

    Gives the column an alias.

    Gives the column an alias.

    // Renames colA to colB in select output.
    df.select($"colA".as('colB))
  21. def as(alias: String): Column

    Gives the column an alias.

    Gives the column an alias.

    // Renames colA to colB in select output.
    df.select($"colA".as("colB"))
  22. final def asInstanceOf[T0]: T0

    Definition Classes
    Any
  23. def asc: Column

    Returns an ordering used in sorting.

    Returns an ordering used in sorting.

    // Scala: sort a DataFrame by age column in ascending order.
    df.sort(df("age").asc)
    
    // Java
    df.sort(df.col("age").asc());
  24. def cast(to: String): Column

    Casts the column to a different data type, using the canonical string representation of the type.

    Casts the column to a different data type, using the canonical string representation of the type. The supported types are: string, boolean, byte, short, int, long, float, double, decimal, date, timestamp.

    // Casts colA to integer.
    df.select(df("colA").cast("int"))
  25. def cast(to: DataType): Column

    Casts the column to a different data type.

    Casts the column to a different data type.

    // Casts colA to IntegerType.
    import org.apache.spark.sql.types.IntegerType
    df.select(df("colA").cast(IntegerType))
    
    // equivalent to
    df.select(df("colA").cast("int"))
  26. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  27. def contains(other: Any): Column

    Contains the other element.

  28. def desc: Column

    Returns an ordering used in sorting.

    Returns an ordering used in sorting.

    // Scala: sort a DataFrame by age column in descending order.
    df.sort(df("age").desc)
    
    // Java
    df.sort(df.col("age").desc());
  29. def divide(other: Any): Column

    Division this expression by another expression.

    Division this expression by another expression.

    // Scala: The following divides a person's height by their weight.
    people.select( people("height") / people("weight") )
    
    // Java:
    people.select( people("height").divide(people("weight")) );
  30. def endsWith(literal: String): Column

    String ends with another string literal.

  31. def endsWith(other: Column): Column

    String ends with.

  32. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  33. def eqNullSafe(other: Any): Column

    Equality test that is safe for null values.

  34. def equalTo(other: Any): Column

    Equality test.

    Equality test.

    // Scala:
    df.filter( df("colA") === df("colB") )
    
    // Java
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").equalTo(col("colB")) );
  35. def equals(that: Any): Boolean

    Definition Classes
    Column → AnyRef → Any
  36. def explain(extended: Boolean): Unit

    Prints the expression to the console for debugging purpose.

  37. val expr: Expression

    Attributes
    protected[org.apache.spark.sql]
  38. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  39. def geq(other: Any): Column

    Greater than or equal to an expression.

    Greater than or equal to an expression.

    // Scala: The following selects people age 21 or older than 21.
    people.select( people("age") >= 21 )
    
    // Java:
    people.select( people("age").geq(21) )
  40. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any
  41. def getField(fieldName: String): Column

    An expression that gets a field by name in a StructField.

  42. def getItem(ordinal: Int): Column

    An expression that gets an item at position ordinal out of an array.

  43. def gt(other: Any): Column

    Greater than.

    Greater than.

    // Scala: The following selects people older than 21.
    people.select( people("age") > lit(21) )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    people.select( people("age").gt(21) );
  44. def hashCode(): Int

    Definition Classes
    Column → AnyRef → Any
  45. def in(list: Column*): Column

    A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

    A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

    Annotations
    @varargs()
  46. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any
  47. def isNotNull: Column

    True if the current expression is NOT null.

  48. def isNull: Column

    True if the current expression is null.

  49. def leq(other: Any): Column

    Less than or equal to.

    Less than or equal to.

    // Scala: The following selects people age 21 or younger than 21.
    people.select( people("age") <= 21 )
    
    // Java:
    people.select( people("age").leq(21) );
  50. def like(literal: String): Column

    SQL like expression.

  51. def lt(other: Any): Column

    Less than.

    Less than.

    // Scala: The following selects people younger than 21.
    people.select( people("age") < 21 )
    
    // Java:
    people.select( people("age").lt(21) );
  52. def minus(other: Any): Column

    Subtraction.

    Subtraction. Subtract the other expression from this expression.

    // Scala: The following selects the difference between people's height and their weight.
    people.select( people("height") - people("weight") )
    
    // Java:
    people.select( people("height").minus(people("weight")) );
  53. def mod(other: Any): Column

    Modulo (a.

    Modulo (a.k.a. remainder) expression.

  54. def multiply(other: Any): Column

    Multiplication of this expression and another expression.

    Multiplication of this expression and another expression.

    // Scala: The following multiplies a person's height by their weight.
    people.select( people("height") * people("weight") )
    
    // Java:
    people.select( people("height").multiply(people("weight")) );
  55. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  56. def notEqual(other: Any): Column

    Inequality test.

    Inequality test.

    // Scala:
    df.select( df("colA") !== df("colB") )
    df.select( !(df("colA") === df("colB")) )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").notEqual(col("colB")) );
  57. final def notify(): Unit

    Definition Classes
    AnyRef
  58. final def notifyAll(): Unit

    Definition Classes
    AnyRef
  59. def or(other: Column): Column

    Boolean OR.

    Boolean OR.

    // Scala: The following selects people that are in school or employed.
    people.filter( people("inSchool") || people("isEmployed") )
    
    // Java:
    people.filter( people("inSchool").or(people("isEmployed")) );
  60. def plus(other: Any): Column

    Sum of this expression and another expression.

    Sum of this expression and another expression.

    // Scala: The following selects the sum of a person's height and weight.
    people.select( people("height") + people("weight") )
    
    // Java:
    people.select( people("height").plus(people("weight")) );
  61. def rlike(literal: String): Column

    SQL RLIKE expression (LIKE with Regex).

  62. def startsWith(literal: String): Column

    String starts with another string literal.

  63. def startsWith(other: Column): Column

    String starts with.

  64. def substr(startPos: Int, len: Int): Column

    An expression that returns a substring.

    An expression that returns a substring.

    startPos

    starting position.

    len

    length of the substring.

  65. def substr(startPos: Column, len: Column): Column

    An expression that returns a substring.

    An expression that returns a substring.

    startPos

    expression for the starting position.

    len

    expression for the length of the substring.

  66. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef
  67. def toString(): String

    Definition Classes
    Column → AnyRef → Any
  68. def unary_!: Column

    Inversion of boolean expression, i.

    Inversion of boolean expression, i.e. NOT. {{ // Scala: select rows that are not active (isActive === false) df.filter( !df("isActive") )

    // Java: import static org.apache.spark.sql.functions.*; df.filter( not(df.col("isActive")) ); }}

  69. def unary_-: Column

    Unary minus, i.

    Unary minus, i.e. negate the expression.

    // Scala: select the amount column and negates all values.
    df.select( -df("amount") )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    df.select( negate(col("amount") );
  70. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  71. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  72. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  73. def ||(other: Any): Column

    Boolean OR.

    Boolean OR.

    // Scala: The following selects people that are in school or employed.
    people.filter( people("inSchool") || people("isEmployed") )
    
    // Java:
    people.filter( people("inSchool").or(people("isEmployed")) );

Inherited from AnyRef

Inherited from Any

df_ops

expr_ops

java_expr_ops

Ungrouped