Source code for pyspark.ml.feature

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
if sys.version > '3':
    basestring = str

from pyspark import since
from pyspark.rdd import ignore_unicode_prefix
from pyspark.ml.param.shared import *
from pyspark.ml.util import keyword_only
from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaTransformer, _jvm
from pyspark.mllib.common import inherit_doc
from pyspark.mllib.linalg import _convert_to_vector

__all__ = ['Binarizer', 'Bucketizer', 'CountVectorizer', 'CountVectorizerModel', 'DCT',
           'ElementwiseProduct', 'HashingTF', 'IDF', 'IDFModel', 'IndexToString', 'MinMaxScaler',
           'MinMaxScalerModel', 'NGram', 'Normalizer', 'OneHotEncoder', 'PCA', 'PCAModel',
           'PolynomialExpansion', 'RegexTokenizer', 'RFormula', 'RFormulaModel', 'SQLTransformer',
           'StandardScaler', 'StandardScalerModel', 'StopWordsRemover', 'StringIndexer',
           'StringIndexerModel', 'Tokenizer', 'VectorAssembler', 'VectorIndexer', 'VectorSlicer',
           'Word2Vec', 'Word2VecModel']


@inherit_doc
[docs]class Binarizer(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental Binarize a column of continuous features given a threshold. >>> df = sqlContext.createDataFrame([(0.5,)], ["values"]) >>> binarizer = Binarizer(threshold=1.0, inputCol="values", outputCol="features") >>> binarizer.transform(df).head().features 0.0 >>> binarizer.setParams(outputCol="freqs").transform(df).head().freqs 0.0 >>> params = {binarizer.threshold: -0.5, binarizer.outputCol: "vector"} >>> binarizer.transform(df, params).head().vector 1.0 .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc threshold = Param(Params._dummy(), "threshold", "threshold in binary classification prediction, in range [0, 1]") @keyword_only def __init__(self, threshold=0.0, inputCol=None, outputCol=None): """ __init__(self, threshold=0.0, inputCol=None, outputCol=None) """ super(Binarizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Binarizer", self.uid) self.threshold = Param(self, "threshold", "threshold in binary classification prediction, in range [0, 1]") self._setDefault(threshold=0.0) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, threshold=0.0, inputCol=None, outputCol=None): """ setParams(self, threshold=0.0, inputCol=None, outputCol=None) Sets params for this Binarizer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setThreshold(self, value): """ Sets the value of :py:attr:`threshold`. """ self._paramMap[self.threshold] = value return self
@since("1.4.0")
[docs] def getThreshold(self): """ Gets the value of threshold or its default value. """ return self.getOrDefault(self.threshold)
@inherit_doc
[docs]class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental Maps a column of continuous features to a column of feature buckets. >>> df = sqlContext.createDataFrame([(0.1,), (0.4,), (1.2,), (1.5,)], ["values"]) >>> bucketizer = Bucketizer(splits=[-float("inf"), 0.5, 1.4, float("inf")], ... inputCol="values", outputCol="buckets") >>> bucketed = bucketizer.transform(df).collect() >>> bucketed[0].buckets 0.0 >>> bucketed[1].buckets 0.0 >>> bucketed[2].buckets 1.0 >>> bucketed[3].buckets 2.0 >>> bucketizer.setParams(outputCol="b").transform(df).head().b 0.0 .. versionadded:: 1.3.0 """ # a placeholder to make it appear in the generated doc splits = \ Param(Params._dummy(), "splits", "Split points for mapping continuous features into buckets. With n+1 splits, " + "there are n buckets. A bucket defined by splits x,y holds values in the " + "range [x,y) except the last bucket, which also includes y. The splits " + "should be strictly increasing. Values at -inf, inf must be explicitly " + "provided to cover all Double values; otherwise, values outside the splits " + "specified will be treated as errors.") @keyword_only def __init__(self, splits=None, inputCol=None, outputCol=None): """ __init__(self, splits=None, inputCol=None, outputCol=None) """ super(Bucketizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Bucketizer", self.uid) #: param for Splitting points for mapping continuous features into buckets. With n+1 splits, # there are n buckets. A bucket defined by splits x,y holds values in the range [x,y) # except the last bucket, which also includes y. The splits should be strictly increasing. # Values at -inf, inf must be explicitly provided to cover all Double values; otherwise, # values outside the splits specified will be treated as errors. self.splits = \ Param(self, "splits", "Split points for mapping continuous features into buckets. With n+1 splits, " + "there are n buckets. A bucket defined by splits x,y holds values in the " + "range [x,y) except the last bucket, which also includes y. The splits " + "should be strictly increasing. Values at -inf, inf must be explicitly " + "provided to cover all Double values; otherwise, values outside the splits " + "specified will be treated as errors.") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, splits=None, inputCol=None, outputCol=None): """ setParams(self, splits=None, inputCol=None, outputCol=None) Sets params for this Bucketizer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setSplits(self, value): """ Sets the value of :py:attr:`splits`. """ self._paramMap[self.splits] = value return self
@since("1.4.0")
[docs] def getSplits(self): """ Gets the value of threshold or its default value. """ return self.getOrDefault(self.splits)
@inherit_doc
[docs]class CountVectorizer(JavaEstimator, HasInputCol, HasOutputCol): """ .. note:: Experimental Extracts a vocabulary from document collections and generates a :py:attr:`CountVectorizerModel`. >>> df = sqlContext.createDataFrame( ... [(0, ["a", "b", "c"]), (1, ["a", "b", "b", "c", "a"])], ... ["label", "raw"]) >>> cv = CountVectorizer(inputCol="raw", outputCol="vectors") >>> model = cv.fit(df) >>> model.transform(df).show(truncate=False) +-----+---------------+-------------------------+ |label|raw |vectors | +-----+---------------+-------------------------+ |0 |[a, b, c] |(3,[0,1,2],[1.0,1.0,1.0])| |1 |[a, b, b, c, a]|(3,[0,1,2],[2.0,2.0,1.0])| +-----+---------------+-------------------------+ ... >>> sorted(map(str, model.vocabulary)) ['a', 'b', 'c'] .. versionadded:: 1.6.0 """ # a placeholder to make it appear in the generated doc minTF = Param( Params._dummy(), "minTF", "Filter to ignore rare words in" + " a document. For each document, terms with frequency/count less than the given" + " threshold are ignored. If this is an integer >= 1, then this specifies a count (of" + " times the term must appear in the document); if this is a double in [0,1), then this " + "specifies a fraction (out of the document's token count). Note that the parameter is " + "only used in transform of CountVectorizerModel and does not affect fitting. Default 1.0") minDF = Param( Params._dummy(), "minDF", "Specifies the minimum number of" + " different documents a term must appear in to be included in the vocabulary." + " If this is an integer >= 1, this specifies the number of documents the term must" + " appear in; if this is a double in [0,1), then this specifies the fraction of documents." + " Default 1.0") vocabSize = Param( Params._dummy(), "vocabSize", "max size of the vocabulary. Default 1 << 18.") @keyword_only def __init__(self, minTF=1.0, minDF=1.0, vocabSize=1 << 18, inputCol=None, outputCol=None): """ __init__(self, minTF=1.0, minDF=1.0, vocabSize=1 << 18, inputCol=None, outputCol=None) """ super(CountVectorizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer", self.uid) self.minTF = Param( self, "minTF", "Filter to ignore rare words in" + " a document. For each document, terms with frequency/count less than the given" + " threshold are ignored. If this is an integer >= 1, then this specifies a count (of" + " times the term must appear in the document); if this is a double in [0,1), then " + "this specifies a fraction (out of the document's token count). Note that the " + "parameter is only used in transform of CountVectorizerModel and does not affect" + "fitting. Default 1.0") self.minDF = Param( self, "minDF", "Specifies the minimum number of" + " different documents a term must appear in to be included in the vocabulary." + " If this is an integer >= 1, this specifies the number of documents the term must" + " appear in; if this is a double in [0,1), then this specifies the fraction of " + "documents. Default 1.0") self.vocabSize = Param( self, "vocabSize", "max size of the vocabulary. Default 1 << 18.") self._setDefault(minTF=1.0, minDF=1.0, vocabSize=1 << 18) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, minTF=1.0, minDF=1.0, vocabSize=1 << 18, inputCol=None, outputCol=None): """ setParams(self, minTF=1.0, minDF=1.0, vocabSize=1 << 18, inputCol=None, outputCol=None) Set the params for the CountVectorizer """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setMinTF(self, value): """ Sets the value of :py:attr:`minTF`. """ self._paramMap[self.minTF] = value return self
@since("1.6.0")
[docs] def getMinTF(self): """ Gets the value of minTF or its default value. """ return self.getOrDefault(self.minTF)
@since("1.6.0")
[docs] def setMinDF(self, value): """ Sets the value of :py:attr:`minDF`. """ self._paramMap[self.minDF] = value return self
@since("1.6.0")
[docs] def getMinDF(self): """ Gets the value of minDF or its default value. """ return self.getOrDefault(self.minDF)
@since("1.6.0")
[docs] def setVocabSize(self, value): """ Sets the value of :py:attr:`vocabSize`. """ self._paramMap[self.vocabSize] = value return self
@since("1.6.0")
[docs] def getVocabSize(self): """ Gets the value of vocabSize or its default value. """ return self.getOrDefault(self.vocabSize)
def _create_model(self, java_model): return CountVectorizerModel(java_model)
[docs]class CountVectorizerModel(JavaModel): """ .. note:: Experimental Model fitted by CountVectorizer. .. versionadded:: 1.6.0 """ @property @since("1.6.0")
[docs] def vocabulary(self): """ An array of terms in the vocabulary. """ return self._call_java("vocabulary")
@inherit_doc
[docs]class DCT(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A feature transformer that takes the 1D discrete cosine transform of a real vector. No zero padding is performed on the input vector. It returns a real vector of the same length representing the DCT. The return vector is scaled such that the transform matrix is unitary (aka scaled DCT-II). More information on `https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II Wikipedia`. >>> from pyspark.mllib.linalg import Vectors >>> df1 = sqlContext.createDataFrame([(Vectors.dense([5.0, 8.0, 6.0]),)], ["vec"]) >>> dct = DCT(inverse=False, inputCol="vec", outputCol="resultVec") >>> df2 = dct.transform(df1) >>> df2.head().resultVec DenseVector([10.969..., -0.707..., -2.041...]) >>> df3 = DCT(inverse=True, inputCol="resultVec", outputCol="origVec").transform(df2) >>> df3.head().origVec DenseVector([5.0, 8.0, 6.0]) .. versionadded:: 1.6.0 """ # a placeholder to make it appear in the generated doc inverse = Param(Params._dummy(), "inverse", "Set transformer to perform inverse DCT, " + "default False.") @keyword_only def __init__(self, inverse=False, inputCol=None, outputCol=None): """ __init__(self, inverse=False, inputCol=None, outputCol=None) """ super(DCT, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.DCT", self.uid) self.inverse = Param(self, "inverse", "Set transformer to perform inverse DCT, " + "default False.") self._setDefault(inverse=False) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, inverse=False, inputCol=None, outputCol=None): """ setParams(self, inverse=False, inputCol=None, outputCol=None) Sets params for this DCT. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setInverse(self, value): """ Sets the value of :py:attr:`inverse`. """ self._paramMap[self.inverse] = value return self
@since("1.6.0")
[docs] def getInverse(self): """ Gets the value of inverse or its default value. """ return self.getOrDefault(self.inverse)
@inherit_doc
[docs]class ElementwiseProduct(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental Outputs the Hadamard product (i.e., the element-wise product) of each input vector with a provided "weight" vector. In other words, it scales each column of the dataset by a scalar multiplier. >>> from pyspark.mllib.linalg import Vectors >>> df = sqlContext.createDataFrame([(Vectors.dense([2.0, 1.0, 3.0]),)], ["values"]) >>> ep = ElementwiseProduct(scalingVec=Vectors.dense([1.0, 2.0, 3.0]), ... inputCol="values", outputCol="eprod") >>> ep.transform(df).head().eprod DenseVector([2.0, 2.0, 9.0]) >>> ep.setParams(scalingVec=Vectors.dense([2.0, 3.0, 5.0])).transform(df).head().eprod DenseVector([4.0, 3.0, 15.0]) .. versionadded:: 1.5.0 """ # a placeholder to make it appear in the generated doc scalingVec = Param(Params._dummy(), "scalingVec", "vector for hadamard product, " + "it must be MLlib Vector type.") @keyword_only def __init__(self, scalingVec=None, inputCol=None, outputCol=None): """ __init__(self, scalingVec=None, inputCol=None, outputCol=None) """ super(ElementwiseProduct, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ElementwiseProduct", self.uid) self.scalingVec = Param(self, "scalingVec", "vector for hadamard product, " + "it must be MLlib Vector type.") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.5.0")
[docs] def setParams(self, scalingVec=None, inputCol=None, outputCol=None): """ setParams(self, scalingVec=None, inputCol=None, outputCol=None) Sets params for this ElementwiseProduct. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.5.0")
[docs] def setScalingVec(self, value): """ Sets the value of :py:attr:`scalingVec`. """ self._paramMap[self.scalingVec] = value return self
@since("1.5.0")
[docs] def getScalingVec(self): """ Gets the value of scalingVec or its default value. """ return self.getOrDefault(self.scalingVec)
@inherit_doc
[docs]class HashingTF(JavaTransformer, HasInputCol, HasOutputCol, HasNumFeatures): """ .. note:: Experimental Maps a sequence of terms to their term frequencies using the hashing trick. >>> df = sqlContext.createDataFrame([(["a", "b", "c"],)], ["words"]) >>> hashingTF = HashingTF(numFeatures=10, inputCol="words", outputCol="features") >>> hashingTF.transform(df).head().features SparseVector(10, {7: 1.0, 8: 1.0, 9: 1.0}) >>> hashingTF.setParams(outputCol="freqs").transform(df).head().freqs SparseVector(10, {7: 1.0, 8: 1.0, 9: 1.0}) >>> params = {hashingTF.numFeatures: 5, hashingTF.outputCol: "vector"} >>> hashingTF.transform(df, params).head().vector SparseVector(5, {2: 1.0, 3: 1.0, 4: 1.0}) .. versionadded:: 1.3.0 """ @keyword_only def __init__(self, numFeatures=1 << 18, inputCol=None, outputCol=None): """ __init__(self, numFeatures=1 << 18, inputCol=None, outputCol=None) """ super(HashingTF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.HashingTF", self.uid) self._setDefault(numFeatures=1 << 18) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.3.0")
[docs] def setParams(self, numFeatures=1 << 18, inputCol=None, outputCol=None): """ setParams(self, numFeatures=1 << 18, inputCol=None, outputCol=None) Sets params for this HashingTF. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@inherit_doc
[docs]class IDF(JavaEstimator, HasInputCol, HasOutputCol): """ .. note:: Experimental Compute the Inverse Document Frequency (IDF) given a collection of documents. >>> from pyspark.mllib.linalg import DenseVector >>> df = sqlContext.createDataFrame([(DenseVector([1.0, 2.0]),), ... (DenseVector([0.0, 1.0]),), (DenseVector([3.0, 0.2]),)], ["tf"]) >>> idf = IDF(minDocFreq=3, inputCol="tf", outputCol="idf") >>> idf.fit(df).transform(df).head().idf DenseVector([0.0, 0.0]) >>> idf.setParams(outputCol="freqs").fit(df).transform(df).collect()[1].freqs DenseVector([0.0, 0.0]) >>> params = {idf.minDocFreq: 1, idf.outputCol: "vector"} >>> idf.fit(df, params).transform(df).head().vector DenseVector([0.2877, 0.0]) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc minDocFreq = Param(Params._dummy(), "minDocFreq", "minimum of documents in which a term should appear for filtering") @keyword_only def __init__(self, minDocFreq=0, inputCol=None, outputCol=None): """ __init__(self, minDocFreq=0, inputCol=None, outputCol=None) """ super(IDF, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid) self.minDocFreq = Param(self, "minDocFreq", "minimum of documents in which a term should appear for filtering") self._setDefault(minDocFreq=0) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, minDocFreq=0, inputCol=None, outputCol=None): """ setParams(self, minDocFreq=0, inputCol=None, outputCol=None) Sets params for this IDF. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setMinDocFreq(self, value): """ Sets the value of :py:attr:`minDocFreq`. """ self._paramMap[self.minDocFreq] = value return self
@since("1.4.0")
[docs] def getMinDocFreq(self): """ Gets the value of minDocFreq or its default value. """ return self.getOrDefault(self.minDocFreq)
def _create_model(self, java_model): return IDFModel(java_model)
[docs]class IDFModel(JavaModel): """ .. note:: Experimental Model fitted by IDF. .. versionadded:: 1.4.0 """
@inherit_doc
[docs]class MinMaxScaler(JavaEstimator, HasInputCol, HasOutputCol): """ .. note:: Experimental Rescale each feature individually to a common range [min, max] linearly using column summary statistics, which is also known as min-max normalization or Rescaling. The rescaled value for feature E is calculated as, Rescaled(e_i) = (e_i - E_min) / (E_max - E_min) * (max - min) + min For the case E_max == E_min, Rescaled(e_i) = 0.5 * (max + min) Note that since zero values will probably be transformed to non-zero values, output of the transformer will be DenseVector even for sparse input. >>> from pyspark.mllib.linalg import Vectors >>> df = sqlContext.createDataFrame([(Vectors.dense([0.0]),), (Vectors.dense([2.0]),)], ["a"]) >>> mmScaler = MinMaxScaler(inputCol="a", outputCol="scaled") >>> model = mmScaler.fit(df) >>> model.transform(df).show() +-----+------+ | a|scaled| +-----+------+ |[0.0]| [0.0]| |[2.0]| [1.0]| +-----+------+ ... .. versionadded:: 1.6.0 """ # a placeholder to make it appear in the generated doc min = Param(Params._dummy(), "min", "Lower bound of the output feature range") max = Param(Params._dummy(), "max", "Upper bound of the output feature range") @keyword_only def __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None): """ __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None) """ super(MinMaxScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid) self.min = Param(self, "min", "Lower bound of the output feature range") self.max = Param(self, "max", "Upper bound of the output feature range") self._setDefault(min=0.0, max=1.0) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, min=0.0, max=1.0, inputCol=None, outputCol=None): """ setParams(self, min=0.0, max=1.0, inputCol=None, outputCol=None) Sets params for this MinMaxScaler. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setMin(self, value): """ Sets the value of :py:attr:`min`. """ self._paramMap[self.min] = value return self
@since("1.6.0")
[docs] def getMin(self): """ Gets the value of min or its default value. """ return self.getOrDefault(self.min)
@since("1.6.0")
[docs] def setMax(self, value): """ Sets the value of :py:attr:`max`. """ self._paramMap[self.max] = value return self
@since("1.6.0")
[docs] def getMax(self): """ Gets the value of max or its default value. """ return self.getOrDefault(self.max)
def _create_model(self, java_model): return MinMaxScalerModel(java_model)
[docs]class MinMaxScalerModel(JavaModel): """ .. note:: Experimental Model fitted by :py:class:`MinMaxScaler`. .. versionadded:: 1.6.0 """
@inherit_doc @ignore_unicode_prefix
[docs]class NGram(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A feature transformer that converts the input array of strings into an array of n-grams. Null values in the input array are ignored. It returns an array of n-grams where each n-gram is represented by a space-separated string of words. When the input is empty, an empty array is returned. When the input array length is less than n (number of elements per n-gram), no n-grams are returned. >>> df = sqlContext.createDataFrame([Row(inputTokens=["a", "b", "c", "d", "e"])]) >>> ngram = NGram(n=2, inputCol="inputTokens", outputCol="nGrams") >>> ngram.transform(df).head() Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b', u'b c', u'c d', u'd e']) >>> # Change n-gram length >>> ngram.setParams(n=4).transform(df).head() Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b c d', u'b c d e']) >>> # Temporarily modify output column. >>> ngram.transform(df, {ngram.outputCol: "output"}).head() Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], output=[u'a b c d', u'b c d e']) >>> ngram.transform(df).head() Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b c d', u'b c d e']) >>> # Must use keyword arguments to specify params. >>> ngram.setParams("text") Traceback (most recent call last): ... TypeError: Method setParams forces keyword arguments. .. versionadded:: 1.5.0 """ # a placeholder to make it appear in the generated doc n = Param(Params._dummy(), "n", "number of elements per n-gram (>=1)") @keyword_only def __init__(self, n=2, inputCol=None, outputCol=None): """ __init__(self, n=2, inputCol=None, outputCol=None) """ super(NGram, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.NGram", self.uid) self.n = Param(self, "n", "number of elements per n-gram (>=1)") self._setDefault(n=2) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.5.0")
[docs] def setParams(self, n=2, inputCol=None, outputCol=None): """ setParams(self, n=2, inputCol=None, outputCol=None) Sets params for this NGram. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.5.0")
[docs] def setN(self, value): """ Sets the value of :py:attr:`n`. """ self._paramMap[self.n] = value return self
@since("1.5.0")
[docs] def getN(self): """ Gets the value of n or its default value. """ return self.getOrDefault(self.n)
@inherit_doc
[docs]class Normalizer(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental Normalize a vector to have unit norm using the given p-norm. >>> from pyspark.mllib.linalg import Vectors >>> svec = Vectors.sparse(4, {1: 4.0, 3: 3.0}) >>> df = sqlContext.createDataFrame([(Vectors.dense([3.0, -4.0]), svec)], ["dense", "sparse"]) >>> normalizer = Normalizer(p=2.0, inputCol="dense", outputCol="features") >>> normalizer.transform(df).head().features DenseVector([0.6, -0.8]) >>> normalizer.setParams(inputCol="sparse", outputCol="freqs").transform(df).head().freqs SparseVector(4, {1: 0.8, 3: 0.6}) >>> params = {normalizer.p: 1.0, normalizer.inputCol: "dense", normalizer.outputCol: "vector"} >>> normalizer.transform(df, params).head().vector DenseVector([0.4286, -0.5714]) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc p = Param(Params._dummy(), "p", "the p norm value.") @keyword_only def __init__(self, p=2.0, inputCol=None, outputCol=None): """ __init__(self, p=2.0, inputCol=None, outputCol=None) """ super(Normalizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Normalizer", self.uid) self.p = Param(self, "p", "the p norm value.") self._setDefault(p=2.0) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, p=2.0, inputCol=None, outputCol=None): """ setParams(self, p=2.0, inputCol=None, outputCol=None) Sets params for this Normalizer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setP(self, value): """ Sets the value of :py:attr:`p`. """ self._paramMap[self.p] = value return self
@since("1.4.0")
[docs] def getP(self): """ Gets the value of p or its default value. """ return self.getOrDefault(self.p)
@inherit_doc
[docs]class OneHotEncoder(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A one-hot encoder that maps a column of category indices to a column of binary vectors, with at most a single one-value per row that indicates the input category index. For example with 5 categories, an input value of 2.0 would map to an output vector of `[0.0, 0.0, 1.0, 0.0]`. The last category is not included by default (configurable via :py:attr:`dropLast`) because it makes the vector entries sum up to one, and hence linearly dependent. So an input value of 4.0 maps to `[0.0, 0.0, 0.0, 0.0]`. Note that this is different from scikit-learn's OneHotEncoder, which keeps all categories. The output vectors are sparse. .. seealso:: :py:class:`StringIndexer` for converting categorical values into category indices >>> stringIndexer = StringIndexer(inputCol="label", outputCol="indexed") >>> model = stringIndexer.fit(stringIndDf) >>> td = model.transform(stringIndDf) >>> encoder = OneHotEncoder(inputCol="indexed", outputCol="features") >>> encoder.transform(td).head().features SparseVector(2, {0: 1.0}) >>> encoder.setParams(outputCol="freqs").transform(td).head().freqs SparseVector(2, {0: 1.0}) >>> params = {encoder.dropLast: False, encoder.outputCol: "test"} >>> encoder.transform(td, params).head().test SparseVector(3, {0: 1.0}) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc dropLast = Param(Params._dummy(), "dropLast", "whether to drop the last category") @keyword_only def __init__(self, dropLast=True, inputCol=None, outputCol=None): """ __init__(self, includeFirst=True, inputCol=None, outputCol=None) """ super(OneHotEncoder, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.OneHotEncoder", self.uid) self.dropLast = Param(self, "dropLast", "whether to drop the last category") self._setDefault(dropLast=True) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, dropLast=True, inputCol=None, outputCol=None): """ setParams(self, dropLast=True, inputCol=None, outputCol=None) Sets params for this OneHotEncoder. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setDropLast(self, value): """ Sets the value of :py:attr:`dropLast`. """ self._paramMap[self.dropLast] = value return self
@since("1.4.0")
[docs] def getDropLast(self): """ Gets the value of dropLast or its default value. """ return self.getOrDefault(self.dropLast)
@inherit_doc
[docs]class PolynomialExpansion(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental Perform feature expansion in a polynomial space. As said in wikipedia of Polynomial Expansion, which is available at `http://en.wikipedia.org/wiki/Polynomial_expansion`, "In mathematics, an expansion of a product of sums expresses it as a sum of products by using the fact that multiplication distributes over addition". Take a 2-variable feature vector as an example: `(x, y)`, if we want to expand it with degree 2, then we get `(x, x * x, y, x * y, y * y)`. >>> from pyspark.mllib.linalg import Vectors >>> df = sqlContext.createDataFrame([(Vectors.dense([0.5, 2.0]),)], ["dense"]) >>> px = PolynomialExpansion(degree=2, inputCol="dense", outputCol="expanded") >>> px.transform(df).head().expanded DenseVector([0.5, 0.25, 2.0, 1.0, 4.0]) >>> px.setParams(outputCol="test").transform(df).head().test DenseVector([0.5, 0.25, 2.0, 1.0, 4.0]) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc degree = Param(Params._dummy(), "degree", "the polynomial degree to expand (>= 1)") @keyword_only def __init__(self, degree=2, inputCol=None, outputCol=None): """ __init__(self, degree=2, inputCol=None, outputCol=None) """ super(PolynomialExpansion, self).__init__() self._java_obj = self._new_java_obj( "org.apache.spark.ml.feature.PolynomialExpansion", self.uid) self.degree = Param(self, "degree", "the polynomial degree to expand (>= 1)") self._setDefault(degree=2) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, degree=2, inputCol=None, outputCol=None): """ setParams(self, degree=2, inputCol=None, outputCol=None) Sets params for this PolynomialExpansion. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setDegree(self, value): """ Sets the value of :py:attr:`degree`. """ self._paramMap[self.degree] = value return self
@since("1.4.0")
[docs] def getDegree(self): """ Gets the value of degree or its default value. """ return self.getOrDefault(self.degree)
@inherit_doc @ignore_unicode_prefix
[docs]class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to split the text (default) or repeatedly matching the regex (if gaps is false). Optional parameters also allow filtering tokens using a minimal length. It returns an array of strings that can be empty. >>> df = sqlContext.createDataFrame([("a b c",)], ["text"]) >>> reTokenizer = RegexTokenizer(inputCol="text", outputCol="words") >>> reTokenizer.transform(df).head() Row(text=u'a b c', words=[u'a', u'b', u'c']) >>> # Change a parameter. >>> reTokenizer.setParams(outputCol="tokens").transform(df).head() Row(text=u'a b c', tokens=[u'a', u'b', u'c']) >>> # Temporarily modify a parameter. >>> reTokenizer.transform(df, {reTokenizer.outputCol: "words"}).head() Row(text=u'a b c', words=[u'a', u'b', u'c']) >>> reTokenizer.transform(df).head() Row(text=u'a b c', tokens=[u'a', u'b', u'c']) >>> # Must use keyword arguments to specify params. >>> reTokenizer.setParams("text") Traceback (most recent call last): ... TypeError: Method setParams forces keyword arguments. .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc minTokenLength = Param(Params._dummy(), "minTokenLength", "minimum token length (>= 0)") gaps = Param(Params._dummy(), "gaps", "whether regex splits on gaps (True) or matches tokens") pattern = Param(Params._dummy(), "pattern", "regex pattern (Java dialect) used for tokenizing") @keyword_only def __init__(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, outputCol=None): """ __init__(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, outputCol=None) """ super(RegexTokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RegexTokenizer", self.uid) self.minTokenLength = Param(self, "minTokenLength", "minimum token length (>= 0)") self.gaps = Param(self, "gaps", "whether regex splits on gaps (True) or matches tokens") self.pattern = Param(self, "pattern", "regex pattern (Java dialect) used for tokenizing") self._setDefault(minTokenLength=1, gaps=True, pattern="\\s+") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, outputCol=None): """ setParams(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, outputCol=None) Sets params for this RegexTokenizer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setMinTokenLength(self, value): """ Sets the value of :py:attr:`minTokenLength`. """ self._paramMap[self.minTokenLength] = value return self
@since("1.4.0")
[docs] def getMinTokenLength(self): """ Gets the value of minTokenLength or its default value. """ return self.getOrDefault(self.minTokenLength)
@since("1.4.0")
[docs] def setGaps(self, value): """ Sets the value of :py:attr:`gaps`. """ self._paramMap[self.gaps] = value return self
@since("1.4.0")
[docs] def getGaps(self): """ Gets the value of gaps or its default value. """ return self.getOrDefault(self.gaps)
@since("1.4.0")
[docs] def setPattern(self, value): """ Sets the value of :py:attr:`pattern`. """ self._paramMap[self.pattern] = value return self
@since("1.4.0")
[docs] def getPattern(self): """ Gets the value of pattern or its default value. """ return self.getOrDefault(self.pattern)
@inherit_doc
[docs]class SQLTransformer(JavaTransformer): """ .. note:: Experimental Implements the transforms which are defined by SQL statement. Currently we only support SQL syntax like 'SELECT ... FROM __THIS__' where '__THIS__' represents the underlying table of the input dataset. >>> df = sqlContext.createDataFrame([(0, 1.0, 3.0), (2, 2.0, 5.0)], ["id", "v1", "v2"]) >>> sqlTrans = SQLTransformer( ... statement="SELECT *, (v1 + v2) AS v3, (v1 * v2) AS v4 FROM __THIS__") >>> sqlTrans.transform(df).head() Row(id=0, v1=1.0, v2=3.0, v3=4.0, v4=3.0) .. versionadded:: 1.6.0 """ # a placeholder to make it appear in the generated doc statement = Param(Params._dummy(), "statement", "SQL statement") @keyword_only def __init__(self, statement=None): """ __init__(self, statement=None) """ super(SQLTransformer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.SQLTransformer", self.uid) self.statement = Param(self, "statement", "SQL statement") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, statement=None): """ setParams(self, statement=None) Sets params for this SQLTransformer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setStatement(self, value): """ Sets the value of :py:attr:`statement`. """ self._paramMap[self.statement] = value return self
@since("1.6.0")
[docs] def getStatement(self): """ Gets the value of statement or its default value. """ return self.getOrDefault(self.statement)
@inherit_doc
[docs]class StandardScaler(JavaEstimator, HasInputCol, HasOutputCol): """ .. note:: Experimental Standardizes features by removing the mean and scaling to unit variance using column summary statistics on the samples in the training set. >>> from pyspark.mllib.linalg import Vectors >>> df = sqlContext.createDataFrame([(Vectors.dense([0.0]),), (Vectors.dense([2.0]),)], ["a"]) >>> standardScaler = StandardScaler(inputCol="a", outputCol="scaled") >>> model = standardScaler.fit(df) >>> model.mean DenseVector([1.0]) >>> model.std DenseVector([1.4142]) >>> model.transform(df).collect()[1].scaled DenseVector([1.4142]) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc withMean = Param(Params._dummy(), "withMean", "Center data with mean") withStd = Param(Params._dummy(), "withStd", "Scale to unit standard deviation") @keyword_only def __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None): """ __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None) """ super(StandardScaler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid) self.withMean = Param(self, "withMean", "Center data with mean") self.withStd = Param(self, "withStd", "Scale to unit standard deviation") self._setDefault(withMean=False, withStd=True) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, withMean=False, withStd=True, inputCol=None, outputCol=None): """ setParams(self, withMean=False, withStd=True, inputCol=None, outputCol=None) Sets params for this StandardScaler. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setWithMean(self, value): """ Sets the value of :py:attr:`withMean`. """ self._paramMap[self.withMean] = value return self
@since("1.4.0")
[docs] def getWithMean(self): """ Gets the value of withMean or its default value. """ return self.getOrDefault(self.withMean)
@since("1.4.0")
[docs] def setWithStd(self, value): """ Sets the value of :py:attr:`withStd`. """ self._paramMap[self.withStd] = value return self
@since("1.4.0")
[docs] def getWithStd(self): """ Gets the value of withStd or its default value. """ return self.getOrDefault(self.withStd)
def _create_model(self, java_model): return StandardScalerModel(java_model)
[docs]class StandardScalerModel(JavaModel): """ .. note:: Experimental Model fitted by StandardScaler. .. versionadded:: 1.4.0 """ @property @since("1.5.0")
[docs] def std(self): """ Standard deviation of the StandardScalerModel. """ return self._call_java("std")
@property @since("1.5.0")
[docs] def mean(self): """ Mean of the StandardScalerModel. """ return self._call_java("mean")
@inherit_doc
[docs]class StringIndexer(JavaEstimator, HasInputCol, HasOutputCol, HasHandleInvalid): """ .. note:: Experimental A label indexer that maps a string column of labels to an ML column of label indices. If the input column is numeric, we cast it to string and index the string values. The indices are in [0, numLabels), ordered by label frequencies. So the most frequent label gets index 0. >>> stringIndexer = StringIndexer(inputCol="label", outputCol="indexed") >>> model = stringIndexer.fit(stringIndDf) >>> td = model.transform(stringIndDf) >>> sorted(set([(i[0], i[1]) for i in td.select(td.id, td.indexed).collect()]), ... key=lambda x: x[0]) [(0, 0.0), (1, 2.0), (2, 1.0), (3, 0.0), (4, 0.0), (5, 1.0)] >>> inverter = IndexToString(inputCol="indexed", outputCol="label2", labels=model.labels()) >>> itd = inverter.transform(td) >>> sorted(set([(i[0], str(i[1])) for i in itd.select(itd.id, itd.label2).collect()]), ... key=lambda x: x[0]) [(0, 'a'), (1, 'b'), (2, 'c'), (3, 'a'), (4, 'a'), (5, 'c')] .. versionadded:: 1.4.0 """ @keyword_only def __init__(self, inputCol=None, outputCol=None, handleInvalid="error"): """ __init__(self, inputCol=None, outputCol=None, handleInvalid="error") """ super(StringIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StringIndexer", self.uid) self._setDefault(handleInvalid="error") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, inputCol=None, outputCol=None, handleInvalid="error"): """ setParams(self, inputCol=None, outputCol=None, handleInvalid="error") Sets params for this StringIndexer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
def _create_model(self, java_model): return StringIndexerModel(java_model)
[docs]class StringIndexerModel(JavaModel): """ .. note:: Experimental Model fitted by StringIndexer. .. versionadded:: 1.4.0 """ @property @since("1.5.0")
[docs] def labels(self): """ Ordered list of labels, corresponding to indices to be assigned. """ return self._java_obj.labels
@inherit_doc
[docs]class IndexToString(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A :py:class:`Transformer` that maps a column of indices back to a new column of corresponding string values. The index-string mapping is either from the ML attributes of the input column, or from user-supplied labels (which take precedence over ML attributes). See L{StringIndexer} for converting strings into indices. .. versionadded:: 1.6.0 """ # a placeholder to make the labels show up in generated doc labels = Param(Params._dummy(), "labels", "Optional array of labels specifying index-string mapping." + " If not provided or if empty, then metadata from inputCol is used instead.") @keyword_only def __init__(self, inputCol=None, outputCol=None, labels=None): """ __init__(self, inputCol=None, outputCol=None, labels=None) """ super(IndexToString, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IndexToString", self.uid) self.labels = Param(self, "labels", "Optional array of labels specifying index-string mapping. If not" + " provided or if empty, then metadata from inputCol is used instead.") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, inputCol=None, outputCol=None, labels=None): """ setParams(self, inputCol=None, outputCol=None, labels=None) Sets params for this IndexToString. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setLabels(self, value): """ Sets the value of :py:attr:`labels`. """ self._paramMap[self.labels] = value return self
@since("1.6.0")
[docs] def getLabels(self): """ Gets the value of :py:attr:`labels` or its default value. """ return self.getOrDefault(self.labels)
[docs]class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A feature transformer that filters out stop words from input. Note: null values from input array are preserved unless adding null to stopWords explicitly. .. versionadded:: 1.6.0 """ # a placeholder to make the stopwords show up in generated doc stopWords = Param(Params._dummy(), "stopWords", "The words to be filtered out") caseSensitive = Param(Params._dummy(), "caseSensitive", "whether to do a case sensitive " + "comparison over the stop words") @keyword_only def __init__(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False): """ __init__(self, inputCol=None, outputCol=None, stopWords=None,\ caseSensitive=false) """ super(StopWordsRemover, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StopWordsRemover", self.uid) self.stopWords = Param(self, "stopWords", "The words to be filtered out") self.caseSensitive = Param(self, "caseSensitive", "whether to do a case " + "sensitive comparison over the stop words") stopWordsObj = _jvm().org.apache.spark.ml.feature.StopWords defaultStopWords = stopWordsObj.English() self._setDefault(stopWords=defaultStopWords) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False): """ setParams(self, inputCol="input", outputCol="output", stopWords=None,\ caseSensitive=false) Sets params for this StopWordRemover. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setStopWords(self, value): """ Specify the stopwords to be filtered. """ self._paramMap[self.stopWords] = value return self
@since("1.6.0")
[docs] def getStopWords(self): """ Get the stopwords. """ return self.getOrDefault(self.stopWords)
@since("1.6.0")
[docs] def setCaseSensitive(self, value): """ Set whether to do a case sensitive comparison over the stop words """ self._paramMap[self.caseSensitive] = value return self
@since("1.6.0")
[docs] def getCaseSensitive(self): """ Get whether to do a case sensitive comparison over the stop words. """ return self.getOrDefault(self.caseSensitive)
@inherit_doc @ignore_unicode_prefix
[docs]class Tokenizer(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental A tokenizer that converts the input string to lowercase and then splits it by white spaces. >>> df = sqlContext.createDataFrame([("a b c",)], ["text"]) >>> tokenizer = Tokenizer(inputCol="text", outputCol="words") >>> tokenizer.transform(df).head() Row(text=u'a b c', words=[u'a', u'b', u'c']) >>> # Change a parameter. >>> tokenizer.setParams(outputCol="tokens").transform(df).head() Row(text=u'a b c', tokens=[u'a', u'b', u'c']) >>> # Temporarily modify a parameter. >>> tokenizer.transform(df, {tokenizer.outputCol: "words"}).head() Row(text=u'a b c', words=[u'a', u'b', u'c']) >>> tokenizer.transform(df).head() Row(text=u'a b c', tokens=[u'a', u'b', u'c']) >>> # Must use keyword arguments to specify params. >>> tokenizer.setParams("text") Traceback (most recent call last): ... TypeError: Method setParams forces keyword arguments. .. versionadded:: 1.3.0 """ @keyword_only def __init__(self, inputCol=None, outputCol=None): """ __init__(self, inputCol=None, outputCol=None) """ super(Tokenizer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Tokenizer", self.uid) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.3.0")
[docs] def setParams(self, inputCol=None, outputCol=None): """ setParams(self, inputCol="input", outputCol="output") Sets params for this Tokenizer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@inherit_doc
[docs]class VectorAssembler(JavaTransformer, HasInputCols, HasOutputCol): """ .. note:: Experimental A feature transformer that merges multiple columns into a vector column. >>> df = sqlContext.createDataFrame([(1, 0, 3)], ["a", "b", "c"]) >>> vecAssembler = VectorAssembler(inputCols=["a", "b", "c"], outputCol="features") >>> vecAssembler.transform(df).head().features DenseVector([1.0, 0.0, 3.0]) >>> vecAssembler.setParams(outputCol="freqs").transform(df).head().freqs DenseVector([1.0, 0.0, 3.0]) >>> params = {vecAssembler.inputCols: ["b", "a"], vecAssembler.outputCol: "vector"} >>> vecAssembler.transform(df, params).head().vector DenseVector([0.0, 1.0]) .. versionadded:: 1.4.0 """ @keyword_only def __init__(self, inputCols=None, outputCol=None): """ __init__(self, inputCols=None, outputCol=None) """ super(VectorAssembler, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorAssembler", self.uid) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, inputCols=None, outputCol=None): """ setParams(self, inputCols=None, outputCol=None) Sets params for this VectorAssembler. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@inherit_doc
[docs]class VectorIndexer(JavaEstimator, HasInputCol, HasOutputCol): """ .. note:: Experimental Class for indexing categorical feature columns in a dataset of [[Vector]]. This has 2 usage modes: - Automatically identify categorical features (default behavior) - This helps process a dataset of unknown vectors into a dataset with some continuous features and some categorical features. The choice between continuous and categorical is based upon a maxCategories parameter. - Set maxCategories to the maximum number of categorical any categorical feature should have. - E.g.: Feature 0 has unique values {-1.0, 0.0}, and feature 1 values {1.0, 3.0, 5.0}. If maxCategories = 2, then feature 0 will be declared categorical and use indices {0, 1}, and feature 1 will be declared continuous. - Index all features, if all features are categorical - If maxCategories is set to be very large, then this will build an index of unique values for all features. - Warning: This can cause problems if features are continuous since this will collect ALL unique values to the driver. - E.g.: Feature 0 has unique values {-1.0, 0.0}, and feature 1 values {1.0, 3.0, 5.0}. If maxCategories >= 3, then both features will be declared categorical. This returns a model which can transform categorical features to use 0-based indices. Index stability: - This is not guaranteed to choose the same category index across multiple runs. - If a categorical feature includes value 0, then this is guaranteed to map value 0 to index 0. This maintains vector sparsity. - More stability may be added in the future. TODO: Future extensions: The following functionality is planned for the future: - Preserve metadata in transform; if a feature's metadata is already present, do not recompute. - Specify certain features to not index, either via a parameter or via existing metadata. - Add warning if a categorical feature has only 1 category. - Add option for allowing unknown categories. >>> from pyspark.mllib.linalg import Vectors >>> df = sqlContext.createDataFrame([(Vectors.dense([-1.0, 0.0]),), ... (Vectors.dense([0.0, 1.0]),), (Vectors.dense([0.0, 2.0]),)], ["a"]) >>> indexer = VectorIndexer(maxCategories=2, inputCol="a", outputCol="indexed") >>> model = indexer.fit(df) >>> model.transform(df).head().indexed DenseVector([1.0, 0.0]) >>> model.numFeatures 2 >>> model.categoryMaps {0: {0.0: 0, -1.0: 1}} >>> indexer.setParams(outputCol="test").fit(df).transform(df).collect()[1].test DenseVector([0.0, 1.0]) >>> params = {indexer.maxCategories: 3, indexer.outputCol: "vector"} >>> model2 = indexer.fit(df, params) >>> model2.transform(df).head().vector DenseVector([1.0, 0.0]) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc maxCategories = Param(Params._dummy(), "maxCategories", "Threshold for the number of values a categorical feature can take " + "(>= 2). If a feature is found to have > maxCategories values, then " + "it is declared continuous.") @keyword_only def __init__(self, maxCategories=20, inputCol=None, outputCol=None): """ __init__(self, maxCategories=20, inputCol=None, outputCol=None) """ super(VectorIndexer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid) self.maxCategories = Param(self, "maxCategories", "Threshold for the number of values a categorical feature " + "can take (>= 2). If a feature is found to have " + "> maxCategories values, then it is declared continuous.") self._setDefault(maxCategories=20) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, maxCategories=20, inputCol=None, outputCol=None): """ setParams(self, maxCategories=20, inputCol=None, outputCol=None) Sets params for this VectorIndexer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setMaxCategories(self, value): """ Sets the value of :py:attr:`maxCategories`. """ self._paramMap[self.maxCategories] = value return self
@since("1.4.0")
[docs] def getMaxCategories(self): """ Gets the value of maxCategories or its default value. """ return self.getOrDefault(self.maxCategories)
def _create_model(self, java_model): return VectorIndexerModel(java_model)
class VectorIndexerModel(JavaModel): """ .. note:: Experimental Model fitted by VectorIndexer. .. versionadded:: 1.4.0 """ @property @since("1.4.0") def numFeatures(self): """ Number of features, i.e., length of Vectors which this transforms. """ return self._call_java("numFeatures") @property @since("1.4.0") def categoryMaps(self): """ Feature value index. Keys are categorical feature indices (column indices). Values are maps from original features values to 0-based category indices. If a feature is not in this map, it is treated as continuous. """ return self._call_java("javaCategoryMaps") @inherit_doc
[docs]class VectorSlicer(JavaTransformer, HasInputCol, HasOutputCol): """ .. note:: Experimental This class takes a feature vector and outputs a new feature vector with a subarray of the original features. The subset of features can be specified with either indices (`setIndices()`) or names (`setNames()`). At least one feature must be selected. Duplicate features are not allowed, so there can be no overlap between selected indices and names. The output vector will order features with the selected indices first (in the order given), followed by the selected names (in the order given). >>> from pyspark.mllib.linalg import Vectors >>> df = sqlContext.createDataFrame([ ... (Vectors.dense([-2.0, 2.3, 0.0, 0.0, 1.0]),), ... (Vectors.dense([0.0, 0.0, 0.0, 0.0, 0.0]),), ... (Vectors.dense([0.6, -1.1, -3.0, 4.5, 3.3]),)], ["features"]) >>> vs = VectorSlicer(inputCol="features", outputCol="sliced", indices=[1, 4]) >>> vs.transform(df).head().sliced DenseVector([2.3, 1.0]) .. versionadded:: 1.6.0 """ # a placeholder to make it appear in the generated doc indices = Param(Params._dummy(), "indices", "An array of indices to select features from " + "a vector column. There can be no overlap with names.") names = Param(Params._dummy(), "names", "An array of feature names to select features from " + "a vector column. These names must be specified by ML " + "org.apache.spark.ml.attribute.Attribute. There can be no overlap with " + "indices.") @keyword_only def __init__(self, inputCol=None, outputCol=None, indices=None, names=None): """ __init__(self, inputCol=None, outputCol=None, indices=None, names=None) """ super(VectorSlicer, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSlicer", self.uid) self.indices = Param(self, "indices", "An array of indices to select features from " + "a vector column. There can be no overlap with names.") self.names = Param(self, "names", "An array of feature names to select features from " + "a vector column. These names must be specified by ML " + "org.apache.spark.ml.attribute.Attribute. There can be no overlap " + "with indices.") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.6.0")
[docs] def setParams(self, inputCol=None, outputCol=None, indices=None, names=None): """ setParams(self, inputCol=None, outputCol=None, indices=None, names=None): Sets params for this VectorSlicer. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.6.0")
[docs] def setIndices(self, value): """ Sets the value of :py:attr:`indices`. """ self._paramMap[self.indices] = value return self
@since("1.6.0")
[docs] def getIndices(self): """ Gets the value of indices or its default value. """ return self.getOrDefault(self.indices)
@since("1.6.0")
[docs] def setNames(self, value): """ Sets the value of :py:attr:`names`. """ self._paramMap[self.names] = value return self
@since("1.6.0")
[docs] def getNames(self): """ Gets the value of names or its default value. """ return self.getOrDefault(self.names)
@inherit_doc @ignore_unicode_prefix
[docs]class Word2Vec(JavaEstimator, HasStepSize, HasMaxIter, HasSeed, HasInputCol, HasOutputCol): """ .. note:: Experimental Word2Vec trains a model of `Map(String, Vector)`, i.e. transforms a word into a code for further natural language processing or machine learning process. >>> sent = ("a b " * 100 + "a c " * 10).split(" ") >>> doc = sqlContext.createDataFrame([(sent,), (sent,)], ["sentence"]) >>> model = Word2Vec(vectorSize=5, seed=42, inputCol="sentence", outputCol="model").fit(doc) >>> model.getVectors().show() +----+--------------------+ |word| vector| +----+--------------------+ | a|[0.09461779892444...| | b|[1.15474212169647...| | c|[-0.3794820010662...| +----+--------------------+ ... >>> model.findSynonyms("a", 2).show() +----+--------------------+ |word| similarity| +----+--------------------+ | b| 0.16782984556103436| | c|-0.46761559092107646| +----+--------------------+ ... >>> model.transform(doc).head().model DenseVector([0.5524, -0.4995, -0.3599, 0.0241, 0.3461]) .. versionadded:: 1.4.0 """ # a placeholder to make it appear in the generated doc vectorSize = Param(Params._dummy(), "vectorSize", "the dimension of codes after transforming from words") numPartitions = Param(Params._dummy(), "numPartitions", "number of partitions for sentences of words") minCount = Param(Params._dummy(), "minCount", "the minimum number of times a token must appear to be included in the " + "word2vec model's vocabulary") @keyword_only def __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, seed=None, inputCol=None, outputCol=None): """ __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, \ seed=None, inputCol=None, outputCol=None) """ super(Word2Vec, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid) self.vectorSize = Param(self, "vectorSize", "the dimension of codes after transforming from words") self.numPartitions = Param(self, "numPartitions", "number of partitions for sentences of words") self.minCount = Param(self, "minCount", "the minimum number of times a token must appear to be included " + "in the word2vec model's vocabulary") self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, seed=None) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.4.0")
[docs] def setParams(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, seed=None, inputCol=None, outputCol=None): """ setParams(self, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, seed=None, \ inputCol=None, outputCol=None) Sets params for this Word2Vec. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.4.0")
[docs] def setVectorSize(self, value): """ Sets the value of :py:attr:`vectorSize`. """ self._paramMap[self.vectorSize] = value return self
@since("1.4.0")
[docs] def getVectorSize(self): """ Gets the value of vectorSize or its default value. """ return self.getOrDefault(self.vectorSize)
@since("1.4.0")
[docs] def setNumPartitions(self, value): """ Sets the value of :py:attr:`numPartitions`. """ self._paramMap[self.numPartitions] = value return self
@since("1.4.0")
[docs] def getNumPartitions(self): """ Gets the value of numPartitions or its default value. """ return self.getOrDefault(self.numPartitions)
@since("1.4.0")
[docs] def setMinCount(self, value): """ Sets the value of :py:attr:`minCount`. """ self._paramMap[self.minCount] = value return self
@since("1.4.0")
[docs] def getMinCount(self): """ Gets the value of minCount or its default value. """ return self.getOrDefault(self.minCount)
def _create_model(self, java_model): return Word2VecModel(java_model)
[docs]class Word2VecModel(JavaModel): """ .. note:: Experimental Model fitted by Word2Vec. .. versionadded:: 1.4.0 """ @since("1.5.0")
[docs] def getVectors(self): """ Returns the vector representation of the words as a dataframe with two fields, word and vector. """ return self._call_java("getVectors")
@since("1.5.0")
[docs] def findSynonyms(self, word, num): """ Find "num" number of words closest in similarity to "word". word can be a string or vector representation. Returns a dataframe with two fields word and similarity (which gives the cosine similarity). """ if not isinstance(word, basestring): word = _convert_to_vector(word) return self._call_java("findSynonyms", word, num)
@inherit_doc
[docs]class PCA(JavaEstimator, HasInputCol, HasOutputCol): """ .. note:: Experimental PCA trains a model to project vectors to a low-dimensional space using PCA. >>> from pyspark.mllib.linalg import Vectors >>> data = [(Vectors.sparse(5, [(1, 1.0), (3, 7.0)]),), ... (Vectors.dense([2.0, 0.0, 3.0, 4.0, 5.0]),), ... (Vectors.dense([4.0, 0.0, 0.0, 6.0, 7.0]),)] >>> df = sqlContext.createDataFrame(data,["features"]) >>> pca = PCA(k=2, inputCol="features", outputCol="pca_features") >>> model = pca.fit(df) >>> model.transform(df).collect()[0].pca_features DenseVector([1.648..., -4.013...]) .. versionadded:: 1.5.0 """ # a placeholder to make it appear in the generated doc k = Param(Params._dummy(), "k", "the number of principal components") @keyword_only def __init__(self, k=None, inputCol=None, outputCol=None): """ __init__(self, k=None, inputCol=None, outputCol=None) """ super(PCA, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.PCA", self.uid) self.k = Param(self, "k", "the number of principal components") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.5.0")
[docs] def setParams(self, k=None, inputCol=None, outputCol=None): """ setParams(self, k=None, inputCol=None, outputCol=None) Set params for this PCA. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.5.0")
[docs] def setK(self, value): """ Sets the value of :py:attr:`k`. """ self._paramMap[self.k] = value return self
@since("1.5.0")
[docs] def getK(self): """ Gets the value of k or its default value. """ return self.getOrDefault(self.k)
def _create_model(self, java_model): return PCAModel(java_model)
[docs]class PCAModel(JavaModel): """ .. note:: Experimental Model fitted by PCA. .. versionadded:: 1.5.0 """
@inherit_doc
[docs]class RFormula(JavaEstimator, HasFeaturesCol, HasLabelCol): """ .. note:: Experimental Implements the transforms required for fitting a dataset against an R model formula. Currently we support a limited subset of the R operators, including '~', '.', ':', '+', and '-'. Also see the R formula docs: http://stat.ethz.ch/R-manual/R-patched/library/stats/html/formula.html >>> df = sqlContext.createDataFrame([ ... (1.0, 1.0, "a"), ... (0.0, 2.0, "b"), ... (0.0, 0.0, "a") ... ], ["y", "x", "s"]) >>> rf = RFormula(formula="y ~ x + s") >>> rf.fit(df).transform(df).show() +---+---+---+---------+-----+ | y| x| s| features|label| +---+---+---+---------+-----+ |1.0|1.0| a|[1.0,1.0]| 1.0| |0.0|2.0| b|[2.0,0.0]| 0.0| |0.0|0.0| a|[0.0,1.0]| 0.0| +---+---+---+---------+-----+ ... >>> rf.fit(df, {rf.formula: "y ~ . - s"}).transform(df).show() +---+---+---+--------+-----+ | y| x| s|features|label| +---+---+---+--------+-----+ |1.0|1.0| a| [1.0]| 1.0| |0.0|2.0| b| [2.0]| 0.0| |0.0|0.0| a| [0.0]| 0.0| +---+---+---+--------+-----+ ... .. versionadded:: 1.5.0 """ # a placeholder to make it appear in the generated doc formula = Param(Params._dummy(), "formula", "R model formula") @keyword_only def __init__(self, formula=None, featuresCol="features", labelCol="label"): """ __init__(self, formula=None, featuresCol="features", labelCol="label") """ super(RFormula, self).__init__() self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid) self.formula = Param(self, "formula", "R model formula") kwargs = self.__init__._input_kwargs self.setParams(**kwargs) @keyword_only @since("1.5.0")
[docs] def setParams(self, formula=None, featuresCol="features", labelCol="label"): """ setParams(self, formula=None, featuresCol="features", labelCol="label") Sets params for RFormula. """ kwargs = self.setParams._input_kwargs return self._set(**kwargs)
@since("1.5.0")
[docs] def setFormula(self, value): """ Sets the value of :py:attr:`formula`. """ self._paramMap[self.formula] = value return self
@since("1.5.0")
[docs] def getFormula(self): """ Gets the value of :py:attr:`formula`. """ return self.getOrDefault(self.formula)
def _create_model(self, java_model): return RFormulaModel(java_model)
[docs]class RFormulaModel(JavaModel): """ .. note:: Experimental Model fitted by :py:class:`RFormula`. .. versionadded:: 1.5.0 """
if __name__ == "__main__": import doctest from pyspark.context import SparkContext from pyspark.sql import Row, SQLContext globs = globals().copy() # The small batch size here ensures that we see multiple batches, # even in these small test examples: sc = SparkContext("local[2]", "ml.feature tests") sqlContext = SQLContext(sc) globs['sc'] = sc globs['sqlContext'] = sqlContext testData = sc.parallelize([Row(id=0, label="a"), Row(id=1, label="b"), Row(id=2, label="c"), Row(id=3, label="a"), Row(id=4, label="a"), Row(id=5, label="c")], 2) globs['stringIndDf'] = sqlContext.createDataFrame(testData) (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS) sc.stop() if failure_count: exit(-1)