source: cows_wps/trunk/cows_wps/utils/filesystem_utils.py @ 5615

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/cows_wps/trunk/cows_wps/utils/filesystem_utils.py@7575
Revision 5615, 6.0 KB checked in by spascoe, 10 years ago (diff)

COWS WPS package copied from
 http://proj.badc.rl.ac.uk/svn/dcip/cows-wps/trunk.

This is a stripped down version of the DDP WPS. Some features are
removed and others have been deactivated until we reimplement them in a
more generic way.

Line 
1import os
2import zipfile
3import time
4import shutil
5import cPickle as pickle
6import subprocess
7
8import logging
9log = logging.getLogger(__name__)
10
11def _getDirectoryContents(dirName):
12    "Returns the full path of all the contents of a given directory"
13    return [os.path.join(dirName, name) for name in os.listdir(dirName)]
14
15def getAllSubfolders(dirName, fullPath=True):
16    """
17    Returns all the subfolder found in a directory, if fullPath=True then
18    the full path is returned not just the directory names.
19    """
20    if fullPath == True:
21        folders = filter(os.path.isdir, _getDirectoryContents(dirName))
22    else:
23        folders = [name for name in os.listdir(dirName) 
24                   if os.path.isdir(os.path.join(dirName, name)) ]
25   
26    return folders
27
28def makeSurePathExists(path, failMessage="%s not found."):
29    """
30    Checks if a path exists, raises and exception if not. The failMessage is the
31    message used by the exception, it should contain one %s that will be replaced
32    by the path.   
33    """
34    if not os.path.exists(path):
35        raise Exception(failMessage % (path,))
36
37def getDiskUsage(path):
38    "Returns the size of a path given by the 'du -sh' command."
39    p = subprocess.Popen(['du', path, '-sh'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
40    p.wait()
41   
42    stderr = p.stderr.read().strip()
43    stdout = p.stdout.read().strip()
44   
45    if stderr:
46        print >> sys.stderr, stderr
47   
48    usage = stdout.split('\t')[0].strip()
49    return usage   
50
51def getSize(path):
52   
53    makeSurePathExists(path)
54   
55    if os.path.isfile(path):
56        return os.path.getsize(path)
57   
58    size = 0
59    for file in getAllFilesInDirectory(path, recursive=True, fullPath=True):
60        size += os.path.getsize(file)
61   
62    return size
63   
64
65def readFile(fileName):
66    "Reads the contents of a file and returns it as one string"
67    fin = open(fileName, 'r')
68    try:
69        content = fin.read()
70    finally:
71        fin.close()
72       
73    return content
74
75def writeFile(fileName, content):
76    fout = open(fileName, 'w')
77   
78    try:
79        fout.write(content)
80    finally:
81        fout.close()
82       
83    return fileName
84
85def readLinesFromFile(path):
86    "Reads the contents of a file and returns it as a list of lines"
87    fin = open(path, 'r')
88    try:
89        lines = fin.readlines()
90    finally:
91        fin.close()
92       
93    return lines
94   
95def readPickledFile(path):
96    "Reads the contents of a pickled file"
97    fin = open(path, 'r')
98    try:
99        content = pickle.load(fin)
100    except:
101        log.exception("problem reading pickle file from path = %s" % (path,))
102        raise
103   
104    finally:
105        fin.close()   
106
107    return content
108
109def writePickledFile(path, content):
110    "Reads the contents of a pickled file"
111   
112    fout = open(path, 'w')
113    try:
114        pickle.dump(content, fout)
115    finally:
116        fout.close()   
117
118def _getAllFilesInDirectoryNonRecursive(path, fullPath=False):
119    contents = [os.path.join(path, x) for x in os.listdir(path)]
120    files = filter(os.path.isfile, contents)
121   
122    if fullPath == True:
123        return files
124    else:
125        return [os.path.basename(x) for x in files]
126
127def _getAllFilesInDirectoryRecursive(path, fullPath=False):
128    l = []
129
130    for currentPath, dirs, files in os.walk(path):
131        for f in files:
132            filePath = os.path.join(currentPath,f)
133           
134            if fullPath == False:
135                filePath = filePath[len(path)+1:]
136               
137            l.append(filePath)
138   
139    return l
140
141def getAllFilesInDirectory(path, recursive=False, fullPath=False):
142    "Reads all the files contained in a directory"
143   
144    if recursive == False:
145        return _getAllFilesInDirectoryNonRecursive(path, fullPath=fullPath)
146    else:
147        return _getAllFilesInDirectoryRecursive(path, fullPath=fullPath)
148
149def deleteDirAndContents(directory):
150   
151    assert directory.find('~') == -1
152    assert directory.find('.') == -1   
153   
154    makeSurePathExists(directory, 'directory %s was not found')
155    assert os.path.isdir(directory)
156   
157    shutil.rmtree(directory)
158   
159def deleteDirectoryContents(directory):
160    assert directory.find('~') == -1
161    assert directory.find('.') == -1   
162
163    makeSurePathExists(directory, 'directory %s was not found')
164    assert os.path.isdir(directory)
165   
166    for path in os.listdir(directory):
167        fullPath = os.path.join(directory, path)
168       
169        if os.path.isfile(fullPath):
170            os.remove(fullPath)
171           
172        elif os.path.isdir(fullPath):
173           
174            deleteDirAndContents(fullPath)
175           
176        else:
177            raise Exception("Cannot delete %s, not a file or a directory." % fullPath)
178       
179def moveDirectoryContents(directory, destination):
180    makeSurePathExists(directory, 'directory %s was not found')
181    assert os.path.isdir(directory)
182   
183    makeSurePathExists(destination, 'destination directory %s was not found')
184    assert os.path.isdir(destination)
185   
186    shutil.move(directory, destination)
187   
188def moveDirectory(directory, destination):
189    makeSurePathExists(directory, 'directory %s was not found')
190    assert os.path.isdir(directory)
191   
192    makeSurePathExists(destination, 'destination directory %s was not found')
193    assert os.path.isdir(destination)
194   
195    directoryName = os.path.basename(directory)
196    destinationDir = os.path.join(destination, directoryName)
197   
198    shutil.move(directory, destinationDir)
199
200def findOrMakeDir(folder, allFolders=True):
201   
202    if not os.path.exists(folder):
203        baseFolder = os.path.dirname(folder)
204         
205        if not os.path.exists(baseFolder):
206            if allFolders:
207                findOrMakeDir(baseFolder, allFolders)
208            else:
209                raise Exception("Can't create folder %s as the folder %s "\
210                                "doesn't exist. Use allFolder=True to create "\
211                                "this folder as well." % ( folder, baseFolder ))
212       
213        os.mkdir(folder)
214    else:
215        assert os.path.isdir(folder)
216
Note: See TracBrowser for help on using the repository browser.