object MLUtils extends Logging
Helper methods to load, save and pre-process data used in MLLib.
- Annotations
- @Since( "0.8.0" )
- Source
- MLUtils.scala
- Alphabetic
- By Inheritance
- MLUtils
- Logging
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
appendBias(vector: Vector): Vector
Returns a new vector with
1.0
(bias) appended to the input vector.Returns a new vector with
1.0
(bias) appended to the input vector.- Annotations
- @Since( "1.0.0" )
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
convertMatrixColumnsFromML(dataset: Dataset[_], cols: String*): DataFrame
Converts matrix columns in an input Dataset to the org.apache.spark.mllib.linalg.Matrix type from the new org.apache.spark.ml.linalg.Matrix type under the
spark.ml
package.Converts matrix columns in an input Dataset to the org.apache.spark.mllib.linalg.Matrix type from the new org.apache.spark.ml.linalg.Matrix type under the
spark.ml
package.- dataset
input dataset
- cols
a list of matrix columns to be converted. Old matrix columns will be ignored. If unspecified, all new matrix columns will be converted except nested ones.
- returns
the input
DataFrame
with new matrix columns converted to the old matrix type
- Annotations
- @Since( "2.0.0" ) @varargs()
-
def
convertMatrixColumnsToML(dataset: Dataset[_], cols: String*): DataFrame
Converts Matrix columns in an input Dataset from the org.apache.spark.mllib.linalg.Matrix type to the new org.apache.spark.ml.linalg.Matrix type under the
spark.ml
package.Converts Matrix columns in an input Dataset from the org.apache.spark.mllib.linalg.Matrix type to the new org.apache.spark.ml.linalg.Matrix type under the
spark.ml
package.- dataset
input dataset
- cols
a list of matrix columns to be converted. New matrix columns will be ignored. If unspecified, all old matrix columns will be converted except nested ones.
- returns
the input
DataFrame
with old matrix columns converted to the new matrix type
- Annotations
- @Since( "2.0.0" ) @varargs()
-
def
convertVectorColumnsFromML(dataset: Dataset[_], cols: String*): DataFrame
Converts vector columns in an input Dataset to the org.apache.spark.mllib.linalg.Vector type from the new org.apache.spark.ml.linalg.Vector type under the
spark.ml
package.Converts vector columns in an input Dataset to the org.apache.spark.mllib.linalg.Vector type from the new org.apache.spark.ml.linalg.Vector type under the
spark.ml
package.- dataset
input dataset
- cols
a list of vector columns to be converted. Old vector columns will be ignored. If unspecified, all new vector columns will be converted except nested ones.
- returns
the input
DataFrame
with new vector columns converted to the old vector type
- Annotations
- @Since( "2.0.0" ) @varargs()
-
def
convertVectorColumnsToML(dataset: Dataset[_], cols: String*): DataFrame
Converts vector columns in an input Dataset from the org.apache.spark.mllib.linalg.Vector type to the new org.apache.spark.ml.linalg.Vector type under the
spark.ml
package.Converts vector columns in an input Dataset from the org.apache.spark.mllib.linalg.Vector type to the new org.apache.spark.ml.linalg.Vector type under the
spark.ml
package.- dataset
input dataset
- cols
a list of vector columns to be converted. New vector columns will be ignored. If unspecified, all old vector columns will be converted except nested ones.
- returns
the input
DataFrame
with old vector columns converted to the new vector type
- Annotations
- @Since( "2.0.0" ) @varargs()
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
kFold(df: DataFrame, numFolds: Int, foldColName: String): Array[(RDD[Row], RDD[Row])]
Version of
kFold()
taking a fold column name.Version of
kFold()
taking a fold column name.- Annotations
- @Since( "3.1.0" )
-
def
kFold[T](rdd: RDD[T], numFolds: Int, seed: Long)(implicit arg0: ClassTag[T]): Array[(RDD[T], RDD[T])]
Version of
kFold()
taking a Long seed.Version of
kFold()
taking a Long seed.- Annotations
- @Since( "2.0.0" )
-
def
kFold[T](rdd: RDD[T], numFolds: Int, seed: Int)(implicit arg0: ClassTag[T]): Array[(RDD[T], RDD[T])]
Return a k element array of pairs of RDDs with the first element of each pair containing the training data, a complement of the validation data and the second element, the validation data, containing a unique 1/kth of the data.
Return a k element array of pairs of RDDs with the first element of each pair containing the training data, a complement of the validation data and the second element, the validation data, containing a unique 1/kth of the data. Where k=numFolds.
- Annotations
- @Since( "1.0.0" )
-
def
loadLabeledPoints(sc: SparkContext, dir: String): RDD[LabeledPoint]
Loads labeled points saved using
RDD[LabeledPoint].saveAsTextFile
with the default number of partitions.Loads labeled points saved using
RDD[LabeledPoint].saveAsTextFile
with the default number of partitions.- Annotations
- @Since( "1.1.0" )
-
def
loadLabeledPoints(sc: SparkContext, path: String, minPartitions: Int): RDD[LabeledPoint]
Loads labeled points saved using
RDD[LabeledPoint].saveAsTextFile
.Loads labeled points saved using
RDD[LabeledPoint].saveAsTextFile
.- sc
Spark context
- path
file or directory path in any Hadoop-supported file system URI
- minPartitions
min number of partitions
- returns
labeled points stored as an RDD[LabeledPoint]
- Annotations
- @Since( "1.1.0" )
-
def
loadLibSVMFile(sc: SparkContext, path: String): RDD[LabeledPoint]
Loads binary labeled data in the LIBSVM format into an RDD[LabeledPoint], with number of features determined automatically and the default number of partitions.
Loads binary labeled data in the LIBSVM format into an RDD[LabeledPoint], with number of features determined automatically and the default number of partitions.
- Annotations
- @Since( "1.0.0" )
-
def
loadLibSVMFile(sc: SparkContext, path: String, numFeatures: Int): RDD[LabeledPoint]
Loads labeled data in the LIBSVM format into an RDD[LabeledPoint], with the default number of partitions.
Loads labeled data in the LIBSVM format into an RDD[LabeledPoint], with the default number of partitions.
- Annotations
- @Since( "1.0.0" )
-
def
loadLibSVMFile(sc: SparkContext, path: String, numFeatures: Int, minPartitions: Int): RDD[LabeledPoint]
Loads labeled data in the LIBSVM format into an RDD[LabeledPoint].
Loads labeled data in the LIBSVM format into an RDD[LabeledPoint]. The LIBSVM format is a text-based format used by LIBSVM and LIBLINEAR. Each line represents a labeled sparse feature vector using the following format:
label index1:value1 index2:value2 ...
where the indices are one-based and in ascending order. This method parses each line into a org.apache.spark.mllib.regression.LabeledPoint, where the feature indices are converted to zero-based.
- sc
Spark context
- path
file or directory path in any Hadoop-supported file system URI
- numFeatures
number of features, which will be determined from the input data if a nonpositive value is given. This is useful when the dataset is already split into multiple files and you want to load them separately, because some features may not present in certain files, which leads to inconsistent feature dimensions.
- minPartitions
min number of partitions
- returns
labeled data stored as an RDD[LabeledPoint]
- Annotations
- @Since( "1.0.0" )
-
def
loadVectors(sc: SparkContext, path: String): RDD[Vector]
Loads vectors saved using
RDD[Vector].saveAsTextFile
with the default number of partitions.Loads vectors saved using
RDD[Vector].saveAsTextFile
with the default number of partitions.- Annotations
- @Since( "1.1.0" )
-
def
loadVectors(sc: SparkContext, path: String, minPartitions: Int): RDD[Vector]
Loads vectors saved using
RDD[Vector].saveAsTextFile
.Loads vectors saved using
RDD[Vector].saveAsTextFile
.- sc
Spark context
- path
file or directory path in any Hadoop-supported file system URI
- minPartitions
min number of partitions
- returns
vectors stored as an RDD[Vector]
- Annotations
- @Since( "1.1.0" )
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
saveAsLibSVMFile(data: RDD[LabeledPoint], dir: String): Unit
Save labeled data in LIBSVM format.
Save labeled data in LIBSVM format.
- data
an RDD of LabeledPoint to be saved
- dir
directory to save the data
- Annotations
- @Since( "1.0.0" )
- See also
org.apache.spark.mllib.util.MLUtils.loadLibSVMFile
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()