Source Code for Module pyspark.files

 1  import os 
 2   
 3   
 4 -class SparkFiles(object): 
 5      """ 
 6      Resolves paths to files added through 
 7      L{SparkContext.addFile()<pyspark.context.SparkContext.addFile>}. 
 8   
 9      SparkFiles contains only classmethods; users should not create SparkFiles 
10      instances. 
11      """ 
12   
13      _root_directory = None 
14      _is_running_on_worker = False 
15      _sc = None 
16   
17 -    def __init__(self): 
18          raise NotImplementedError("Do not construct SparkFiles objects") 
19   
20      @classmethod 
21 -    def get(cls, filename): 
22          """ 
23          Get the absolute path of a file added through C{SparkContext.addFile()}. 
24          """ 
25          path = os.path.join(SparkFiles.getRootDirectory(), filename) 
26          return os.path.abspath(path) 
27   
28      @classmethod 
29 -    def getRootDirectory(cls): 
30          """ 
31          Get the root directory that contains files added through 
32          C{SparkContext.addFile()}. 
33          """ 
34          if cls._is_running_on_worker: 
35              return cls._root_directory 
36          else: 
37              # This will have to change if we support multiple SparkContexts: 
38              return cls._sc._jvm.spark.SparkFiles.getRootDirectory() 
39