source: TI03-DataExtractor/trunk/pydxs/common.py @ 794

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/pydxs/common.py@794
Revision 794, 9.3 KB checked in by astephen, 13 years ago (diff)

Unstable but latest version with multi-variable support and split hooks
for CDML and CSML.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6common.py
7=========
8
9Holds common functions and classes used in the package.
10
11"""
12
13# Import python modules
14import os, re
15
16# Import package modules
17from serverConfig import *
18
19# Set up commmon variables. These are CAPITALISED to
20# improve visibility in other modules.
21
22STAGES=["DatasetGroupPage", "DatasetPage", "VariablesPage", "DomainPage", "ResultsPage"]
23
24DOMAIN_PARAMS=("start_time", "end_time", "time_interval", "horiz_domain", 
25                      "resolution", "vertical_units", "vertical_domain", 
26                       "outputFormat")
27
28TIME_KEYS=["Year", "Month", "Day", "Hour", "Minute", "Second"]
29
30HORIZ_KEYS=["northernExtent", "easternExtent", "southernExtent", "westernExtent"]
31
32CF_METADATA_GLOBAL_ATTRIBUTE_KEYS="Conventions title source institution history references comment".split()
33
34# Set up common functions
35
36def createTimeKeyList():
37    """
38    Returns all the separate date and time component arguments required as a list.
39    """
40    allTimeKeys=[]
41    for ts in ["start", "end"]:
42        for i in range(1, MAX_NUM_DATASETS+1):
43            for key in TIME_KEYS:
44                allTimeKeys.append("%s%s_%s" % (ts, key, i))
45    return allTimeKeys
46
47
48def isUndefined(dict, item): 
49    """
50    Function that returns 1 if item is not a key in dict or
51    is defined as None in dict.
52    """
53    if dict.has_key(item):
54        if dict[item]!=None:
55            return 0
56    return 1
57
58
59def keyPatternMatch(dct, pattern):
60    """
61    Returns 1 if one or more keys in the dictionary 'dct' match the
62    pattern provided using string.find(). Returns 0 otherwise.
63    """
64    for key in dct.keys():
65        if key.find(pattern)>-1:
66            return 1
67    return 0
68
69
70def getValuesInRange(start, end, array):
71    """
72    Takes a start and end value and returns the values in the array that are between them.
73    If not in range and are the same value then returns [start].
74    """
75    # check all are floats
76    array=map(lambda x: float(x), list(array))
77    if array[0]>array[-1]: array.reverse()
78    if start>end:  (start, end)=(end, start)
79    (start, end)=(float(start), float(end))
80    rtarray=[]
81    for i in array:
82        if i>=start and i<=end:
83            rtarray.append(i)
84    if rtarray==[] and start==end:
85        rtarray=[start]     
86    return rtarray
87
88
89def getSortedKeysLike(dct, pattern):
90    """
91    Returns a list of all keys in the dictionary 'dct' that do a
92    string match on 'pattern'.
93    """
94    rtlist=[]
95    for key in dct.keys():
96        if key.find(pattern)>-1:
97            rtlist.append(key)
98    rtlist.sort()       
99    return rtlist
100   
101 
102def getDictSubsetMatching(dct, pattern):
103    """
104    Returns a dictionary of all items in input dictionary 'dct'
105    where keys match 'pattern'.
106    """
107    rtdict={}
108    for key in dct.keys():
109        if key.find(pattern)>-1:
110            rtdict[key]=dct[key]
111           
112    return rtdict
113
114
115def intAll(stringList):
116    """
117    Takes a list of strings and returns them as ints.
118    """
119    return [int(i) for i in stringList]
120
121
122def convertDateTimeStringToYYYYMMDDHH(timeString):
123    """
124    Takes in a long CF-compliant time string and returns a shorter
125    YYYYMMDDHH string.
126    """
127    match=re.match(r"(\d{4}).(\d{1,2}).(\d{1,2})(\s+|T)(\d+):", timeString)
128    if match:
129        (y,m,d,blank,h)=match.groups()
130        timeString="%.4d%.2d%.2d%.2d" % (int(y), int(m), int(d), int(h))
131       
132    return timeString
133
134
135def deUnicodeObject(obj):
136    """
137    Returns the object identical except unicode strings are all
138    converted to normal strings.
139    """
140    tupleFound=None
141    if type(obj)==type((1,2)): 
142        tupleFound=1
143        obj=list(obj)
144
145    if type(obj)==type([]):
146        rtobj=[]
147        for i in obj:
148            if type(i) in (type([]), type((1,2))):
149                rtobj.append(deUnicodeObject(i)) 
150            elif type(i)==type(u""):
151                rtobj.append(str(i))
152            else:
153                rtobj.append(i)
154        if tupleFound==1: rtobj=tuple(rtobj)
155
156    elif type(obj)==type(u""):
157        rtobj=str(obj)
158    else: 
159        rtobj=obj
160    return rtobj
161
162
163def translateURI(uri):
164    """
165    Takes a URL and returns the location of the file on the local network
166    or takes a local path and returns the URL of the file/directory.
167    """
168    if uri.find("http")>-1:
169        newpath=uri.replace(OUTPUT_DIR_URL, OUTPUT_DIR)
170    else:
171        newpath=uri.replace(OUTPUT_DIR, OUTPUT_DIR_URL)
172    return newpath
173
174
175
176def overlap(list1, list2):
177    """
178    overlap function - returns a list of overlapping items in list1 and list2.
179    Otherwise returns None.
180    """ 
181    rtlist=[]
182    for i in list1:
183       if i in list2: rtlist.append(i)
184
185    if len(rtlist)>0:
186        return rtlist
187    else:
188        return None
189
190
191def compareAxes(ax1, ax2):
192    """Takes 2 cdms axis objects returning 1 if they are essentially
193    the same and 0 if not."""
194    import cdms
195    for axtype in ("time", "level", "latitude", "longitude"):
196        if cdms.axisMatches(ax1, axtype) and not cdms.axisMatches(ax2, axtype):
197            return 0
198
199    # Check ids
200    if ax1.id!=ax2.id: return 0
201    # Check lengths
202    if len(ax1)!=len(ax2): return 0
203    # Check values
204    if ax1._data_!=ax2._data_: return 0
205    # Check units
206    if ax1.units!=ax2.units: return 0
207    # OK, I think they are the same axis!
208    return 1
209
210
211def compareGrids(grid1, grid2):
212    """Takes 2 cdms grid objects returning 1 if they are essentially
213    the same and 0 if not."""
214    import cdms
215    if compareAxes(grid1.getLatitude(), grid2.getLatitude())==0: return 0
216    if compareAxes(grid1.getLongitude(), grid2.getLongitude())==0: return 0
217    return 1
218   
219
220def nudgeSingleValuesToAxisValues(value, axisValues, axisType):
221        """
222        Takes a value and checks if it is in the axisValues array. If not, it nudges the
223        value to the nearest neighbour in axis. It returns the new value twice along
224        with a message describing the change.
225        """
226        import cdms
227        rtMessage=""
228        newValue=None
229       
230        if value in axisValues:
231            newValue=value
232        else:
233            sortedAxis=[]
234            for i in axisValues:
235                sortedAxis.append(i)
236            sortedAxis.sort()
237           
238            if value<sortedAxis[0]:
239                newValue=sortedAxis[0]
240            elif value>sortedAxis[-1]:
241                newValue=sortedAxis[-1]
242            else:
243                for i in range(len(axisValues)):
244                    if i<(len(axisValues)-1):
245                        (current, nextone)=(axisValues[i], axisValues[i+1])
246                        if current>nextone:
247                            tempc=nextone
248                            nextone=current
249                            current=tempc
250                   
251                        if value>current and value<nextone:
252                            lowergap=value-current
253                            uppergap=nextone-value
254                            if uppergap==lowergap:
255                                newValue=nextone
256                            elif uppergap>lowergap:
257                                newValue=current
258                            elif uppergap<lowergap:
259                                newValue=nextone                       
260                            break
261            if newValue==None:
262                raise DXProcessingError, ("Could not nudge selected value '%s' into axis '%s'." % (value, axisType))
263            rtMessage="%s axis selected value '%s' nudged to nearest value in real axis '%s' ;" % (axisType, value, newValue)                   
264            print rtMessage
265           
266        return (newValue, newValue, rtMessage)   
267
268
269def makeDirsAndPerms(basedir, dirs, permissions, owner, verbose="no"):
270    """
271    A function for making directories recursively and setting permissions/ownership.
272    """
273    if type(dirs)==str: dirs=[dirs]
274    dirs=[basedir]+list(dirs)
275    dir=None
276
277    while len(dirs)>0:
278
279        if dir:
280            dir=os.path.join(dir, dirs[0])
281        else:
282            dir=dirs[0]
283        dirs=dirs[1:]
284
285        if not os.path.exists(dir):
286            if verbose=="yes":   print "Making directory:", dir
287            os.mkdir(dir)
288            os.chmod(dir, permissions)
289            os.system('/bin/chown %s %s' % (owner, dir))
290
291    return
292   
293   
294def checkSubDirectory(user=None):
295    """
296    checkSubDirectory method - checks if the required sub-directory exists
297    to write the output and if not it creates it.
298    """ 
299    if user==None or user=="None":   user="anonymous"
300    outputDir=os.path.join(OUTPUT_DIR, user, "dx_output")
301    if not os.path.isdir(outputDir):
302        makeDirsAndPerms(OUTPUT_DIR, (user, "dx_output"), OUTPUT_DIR_PERMISSION, "%s.%s" %
303                                           (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP))
304    # Local rule follows
305    if LOCAL_RULES==1:  makeAccessControlFile(user)     
306    return outputDir
307
308
309def fixFilePermissions(path):
310    """
311    Fixes permissions on a file.
312    """
313    os.chmod(path, OUTPUT_FILE_PERMISSION)
314    os.system("chown %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, path))   
315   
316   
317# Set up common classes
318class RedirectStdout:
319    """
320    RedirectStdout class - used to direct standard output away from
321    the screen in CGI scripts.
322    """
323
324    def write(self, item):
325        """
326        write method - allows dummy standard out to work.
327        """
328        pass
329
330    def flush(self):
331        """
332        Method to do nothing, again!
333        """
334        pass
Note: See TracBrowser for help on using the repository browser.