source: nappy/trunk/localRules/aircraftData.py @ 368

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/localRules/aircraftData.py@368
Revision 368, 6.6 KB checked in by astephen, 15 years ago (diff)

Updates for correctly converting and reducing volume of FAAM flight data using the localRules/aircraftData.py module.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6airCraftData.py
7===============
8
9Container module for AircraftData class that deals with
10flight data which has unusual time axes and flags in separate
11variables. This class also does sub-sampling by averaging to
121Hz frequency.
13
14
15"""
16
17# Imports from python standard library
18import os,sys
19import MA, Numeric, cdms, MV, cdtime
20
21# Imports from local package
22
23
24class AircraftData:
25    """
26    AircraftData class that deals with
27    flight data which has unusual time axes and flags in separate
28    variables. This class also does sub-sampling by averaging to
29    1Hz frequency.
30    """
31
32    def __init__(self, var, timeVar, flagVar=None, args=["av"]):
33        """
34        Parses args and calls required methods.
35        """
36        if args[0]=="av":
37            samplingMethod="averaging"
38        elif args[0]=="ss":
39            samplingMethod="selection"
40
41        flagsToUse=[0,1]
42        samplingRate=1
43        if len(args)>1:
44            samplingRate=int(args[1])
45
46            if len(args)>2:
47                flagsToUse=[int(flag[0]) for flag in args[2:]]
48
49        if samplingMethod=="averaging":
50            self.subSampledVar=self._subSampleByAveraging(var, timeVar, flagVar,
51                    samplingRate, flagsToUse)
52        elif samplingMethod=="selection":
53            self.subSampledVarself._subSampleBySelection(var, timeVar, flagVar,
54                    samplingRate, flagsToUse)
55
56
57    def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse):
58        """
59        Returns a new variable which is 'var' sub-sampled by averaging
60        at the given samplingRate with data selected according to flagVar
61        including the flag values specified in flagsToUse (defaults are 0 and 1).
62        """
63        maskedArray=self._getMaskedArray(var, flagVar, flagsToUse)
64        shape=var.shape
65        if shape[1]==1:
66            newNumArray=MV.ravel(maskedArray)
67            newArrayMask=MV.ravel(maskedArray.mask())
68            newArray=MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value())
69        else:
70            newArray=Numeric.zeros(shape[0], 'f')
71            for t0 in range(shape[0]):
72                # Set as missing if less than half are valid                 
73                t1Array=maskedArray[t0]
74
75                if samplingRate==1:
76                    # If half or more are good values then calculate the mean
77                    if t1Array.count()>=(shape[1]/2.):
78                        newArray[t0]=MA.average(t1Array)
79                    # otherwise set as missing value
80                    else:
81                        newArray[t0]=maskedArray.fill_value()
82       
83                else:
84                    raise "Averaging for non 1Hz sampling rates not yet supported!"
85
86        # Now re-construct variable axes etc
87        newTimeAxis=self._flatten2DTimeAxis(timeVar, samplingRate)
88        newVar=self._recreateVariable(var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value()) 
89        return newVar
90
91
92    def _flatten2DTimeAxis(self, timevar, samplingRate):
93        """
94        Returns a flattened 2D time axis.
95        """
96        if samplingRate!=1:
97            raise "Cannot yet deal with sub-sampling to non 1Hz sampling rates!"
98
99        timevalues=timevar._data
100        timeunit=timevar.units
101        newTimeValues=[]
102        rate=samplingRate
103
104        for t in timevalues:
105            for i in range(rate):
106                tvalue=t+((1./rate)*i)
107                newTimeValues.append(tvalue)
108
109        newTimeAx=cdms.createAxis(newTimeValues)
110        newTimeAx.units=timeunit
111        newTimeAx.designateTime()
112        newTimeAx.id=newTimeAx.long_name=newTimeAx.standard_name="time"
113        return newTimeAx
114
115
116    def _recreateVariable(self, var, newArray, timeAxis, flagVar, flagMax, missingValue):
117        """
118        Rebuilds a variable using new array and new time axis.
119        """
120        atts=var.attributes
121        newAtts={}
122        for att,value in atts.items():
123            if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value)==1:
124                value=value[0]
125            if type(value) in (type(1), type(1.), type(long(1))):
126                newAtts[att]=value
127            elif type(value)==type(""):
128                newAtts[att]=value.strip()
129
130        # Now create the variable
131        missing=missingValue
132        data=newArray
133        newvar=cdms.createVariable(data, id=var.id, fill_value=missing, axes=[timeAxis], attributes=newAtts)
134        newvar.frequency=1
135        newvar.comment="Data re-sampled to 1Hz frequency by averaging. "
136        if flagVar:
137            newvar.comment=newvar.comment+"Values were included where the flag variable were less than %s and where both the flag value and actual data value were present (i.e. not missing values)." % (flagMax+1)
138        else:
139            newvar.comment=newvar.comment+"The flag variable was not used in creating this variable."
140        newvar.comment=newvar.comment+" Average data values were set to missing where more than half the values in any one second period were already missing."
141        return newvar
142 
143
144    def _getMaskedArray(self, var, flagVar=None, flagValues=(0,1), missingValuesToTest=(-9999., -32767.)):
145        """
146        Returns a masked array that has the values only present where the flag
147        values are acceptable and the data itself is not missing.
148        """
149        if flagVar:
150            flagFillValue=-1
151            flagGTValue=max(flagValues)
152
153            flagMaskWhereUnacceptable=MA.masked_greater(flagVar, flagGTValue).mask()
154            flagMaskWhereMissing=MA.masked_equal(flagVar, flagFillValue).mask()
155            flagMask=flagMaskWhereUnacceptable+flagMaskWhereMissing
156
157        for fv in missingValuesToTest:
158            if fv in MV.ravel(var):
159                print "Setting missing value for '%s' as: %s" % (var.id, fv)
160                varFillValue=fv
161        else:
162            varFillValue=missingValuesToTest[0]
163
164        if flagVar:
165            varMask=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask()
166            fullmask=MA.bitwise_or(varMask, flagMask)
167            maskedArray=MA.masked_array(var, mask=fullmask, fill_value=varFillValue)
168        else:
169            maskedArray=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue)
170
171        #maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue)
172        #maskedArray.missing_value=varFillValue
173        return maskedArray
174
Note: See TracBrowser for help on using the repository browser.