org.apache.spark.streaming.util

Class HdfsUtils

Object

org.apache.spark.streaming.util.HdfsUtils

public class HdfsUtils
extends Object

Constructor Summary

Constructors

Constructor and Description
HdfsUtils()

Method Summary

Modifier and Type	Method and Description
static boolean	checkFileExists(String path, org.apache.hadoop.conf.Configuration conf) Check if the file exists at the given path.
static void	checkState(boolean state, scala.Function0<String> errorMsg)
static String[]	getFileSegmentLocations(String path, long offset, long length, org.apache.hadoop.conf.Configuration conf) Get the locations of the HDFS blocks containing the given file segment.
static org.apache.hadoop.fs.FileSystem	getFileSystemForPath(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.fs.FSDataInputStream	getInputStream(String path, org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.fs.FSDataOutputStream	getOutputStream(String path, org.apache.hadoop.conf.Configuration conf)

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HdfsUtils

public HdfsUtils()

Method Detail

getOutputStream

public static org.apache.hadoop.fs.FSDataOutputStream getOutputStream(String path,
                                                                      org.apache.hadoop.conf.Configuration conf)

getInputStream

public static org.apache.hadoop.fs.FSDataInputStream getInputStream(String path,
                                                                    org.apache.hadoop.conf.Configuration conf)

checkState

public static void checkState(boolean state,
                              scala.Function0<String> errorMsg)

getFileSegmentLocations

public static String[] getFileSegmentLocations(String path,
                                               long offset,
                                               long length,
                                               org.apache.hadoop.conf.Configuration conf)

Get the locations of the HDFS blocks containing the given file segment.

getFileSystemForPath

public static org.apache.hadoop.fs.FileSystem getFileSystemForPath(org.apache.hadoop.fs.Path path,
                                                                   org.apache.hadoop.conf.Configuration conf)

checkFileExists

public static boolean checkFileExists(String path,
                                      org.apache.hadoop.conf.Configuration conf)

Check if the file exists at the given path.