Changeset 2095 for TI02-CSML


Ignore:
Timestamp:
02/02/07 10:56:19 (13 years ago)
Author:
domlowe
Message:

more bug fixing/tidying in subsetting code

Location:
TI02-CSML/trunk/csml
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • TI02-CSML/trunk/csml/API/__init__.py

    r1932 r2095  
    3333 
    3434##AbstractFeature methods: 
    35 addMethods(csml.parser.AbstractFeature, [ops_AbstractFeature.testmethod, ops_AbstractFeature.getAllowedSubsettings, ops_AbstractFeature._identifyTimeAxis]) 
     35addMethods(csml.parser.AbstractFeature, [ops_AbstractFeature.testmethod, ops_AbstractFeature.getAllowedSubsettings]) 
    3636 
    3737##PointFeature methods: 

  • TI02-CSML/trunk/csml/API/csmlutils.py

    r2081 r2095  
    1515def fixLongitude(request, kwargs): 
    1616    for key in request.keys(): 
    17         if key == 'Lon': #how do we test if it is longitude properly? 
     17        if key == 'longitude': #how do we test if it is longitude properly? 
    1818            for val in request[key]: 
    1919                if val < 0: 

  • TI02-CSML/trunk/csml/API/genSubset.py

    r2091 r2095  
    66 
    77 
    8 def _subsetDomain(timeaxis,times, domain,**kwargs): 
     8def subsetDomain(timeaxis,times, domain,**kwargs): 
    99    '''takes the domain and returns a subset according to the keyword selection criteria 
    1010    time = name of time dimension 
    … …  
    7979    times=timeSelection 
    8080 
    81     tone=csml.API.ops_AbstractFeature.__getCDtime(times[0]) 
     81    tone=csml.csmllibs.csmltime.getCDtime(times[0]) 
    8282    if len(times) == 2: 
    8383        #then this is a range of times (t1, tn), and not a list  
    8484        try: 
    85             tone=csml.API.ops_AbstractFeature.__getCDtime(times[0]) 
     85            tone=csml.csmllibs.csmltime.getCDtime(times[0]) 
    8686        except: 
    8787            tone=times[0] 
    8888        try: 
    89             ttwo=csml.API.ops_AbstractFeature.__getCDtime(times[1]) 
     89            ttwo=csml.csmllibs.csmltime.getCDtime(times[1]) 
    9090        except: 
    9191            ttwo=times[1] 
    9292        times=[]          
    9393        for time in domain[timeName]: 
    94             timeok=csml.API.ops_AbstractFeature.__compareTimes(tone,time,ttwo) 
     94            timeok=csml.csmllibs.csmltime.compareTimes(tone,time,ttwo) 
    9595            if timeok ==1: 
    9696                times.append(time) 
    … …  
    103103            return len(domain[timeName])/numFiles 
    104104 
     105def getCoordTransformTable(domainSubset, crs): 
     106    cTT=csml.parser.GridCoordinatesTable() 
     107    ords =[] 
     108    for key in domainSubset.keys(): 
     109        go=csml.parser.GridOrdinateDescription() 
     110        go.coordAxisLabel=csml.parser.csString(key) 
     111        go.gridAxesSpanned=csml.parser.csString(key) 
     112        go.coordAxisValues = csml.parser.SpatialOrTemporalPositionList() 
     113        if key==crs.axes[crs.timeAxis]: 
     114            go.coordAxisValues.timePositionList=csml.parser.csString(domainSubset[key]) #self.domain.key placeholder 
     115        else: 
     116            go.coordAxisValues.coordinateList=csml.parser.csString(domainSubset[key]) #self.domain.key placeholder 
     117        ords.append(go) #go needs a few more properties setting 
     118    cTT.gridOrdinates=ords 
     119    return cTT 
    105120 
    106 def _genericSubset(feature, csmlpath, ncpath, domain, kwargs): 
     121 
     122 
     123def getTheData(feature, selection, times,timeName): 
     124    domain = feature.domain 
     125    value=feature.value 
     126    files=[] 
     127    strTimes='' 
     128    fulldata=[] 
     129    #get the ratio of times to files 
     130    timeToFileRatio = _getTimeToFileRatio(feature, domain, timeName) 
     131         
     132             
     133    #list to keep track of files that have already been fetched. eg. if multiple times are in a single file only need to get data from that file once... 
     134    filesFetched=[] 
     135     
     136     
     137    for time in times: 
     138        listPosition=domain[timeName].index(time) 
     139        strTimes= strTimes + ' ' + time 
     140        for comp in csmlutils.listify(value.rangeSet.valueArray.valueComponent.quantityList.__insertedExtract.components):  
     141            filePos=int(float(listPosition)/timeToFileRatio) 
     142            if filePos in filesFetched: 
     143                continue #already got data from this file, try next time 
     144            data=comp.getData(fileposition=filePos, **selection) 
     145            files.append(comp.fileList.fileNames.CONTENT[filePos]) #TODO, get the right file name 
     146            if fulldata ==[]: 
     147                fulldata = data.tolist() 
     148            else: 
     149                for item in data.tolist(): 
     150                    fulldata.append(item) 
     151            filesFetched.append(filePos) 
     152        axisorder = data.getAxisIds()  #will need later! 
     153   
     154    return strTimes, axisorder, fulldata 
     155 
     156def genericSubset(feature, csmlpath, ncpath, domain, kwargs): 
    107157    if ncpath is not None: 
    108158        pathToSubsetNetCDF=ncpath 

  • TI02-CSML/trunk/csml/API/ops_AbstractFeature.py

    r2083 r2095  
    1212    return ['none'] 
    1313 
    14 def _identifyTimeAxis(self,listToCheck):  
    15     #given a list of possible axes, find the one which is most probably the time axis. 
    16     for name in listToCheck: 
    17         if string.upper(name) in ['T', 'TIME']: 
    18             tname =name 
    19     return tname 
    20  
    21 def __compareTimes(timemin, timecheck, timemax): 
    22     timeok=0 
    23     try: 
    24         year=int(timecheck.split('-')[0]) 
    25         month=int(timecheck.split('-')[1]) 
    26         day=int(timecheck.split('-')[2].split('T')[0]) 
    27         tcheck=cdtime.comptime(year,month,day) 
    28         timeok=0 
    29         if tcheck.cmp(timemin) >= 0: 
    30             if tcheck.cmp(timemax) < 1: 
    31                 timeok = 1 
    32         print 'TMIN time  %s'%timemin 
    33         print 'CHECK time  %s'%tcheck 
    34         print 'TMAX time  %s'%timemax 
    35     except: 
    36         #if it's not a proper internet time e.g. time = '6' then just compare numerically 
    37         timeok=0 
    38         if timecheck >=timemin: 
    39             if timecheck <=timemax: 
    40                 timeok = 1 
    41     return timeok 
    42  
    43 def __getCDtime(time):  
    44     year=int(time.split('-')[0]) 
    45     month=int(time.split('-')[1]) 
    46     day=int(time.split('-')[2].split('T')[0]) 
    47     hour=int(time.split('-')[2].split('T')[1].split(':')[0]) 
    48     minute=int(time.split('-')[2].split('T')[1].split(':')[1]) 
    49     second=float(time.split('-')[2].split('T')[1].split(':')[2]) 
    50     cdTimeVal=cdtime.comptime(year,month,day,hour,minute,second) 
    51     return cdTimeVal 
    5214 
    5315 
    5416 
    5517 
     18 

  • TI02-CSML/trunk/csml/API/ops_GridSeriesFeature.py

    r2091 r2095  
    4444     
    4545    #non-feature specific setup code, mainly handles the time dimension/calendar 
    46     pathToSubsetNetCDF, kwargs, timeName, calunits, caltype, times=csml.API.genSubset._genericSubset(self, csmlpath, ncpath, self.domain, kwargs) 
     46    pathToSubsetNetCDF, kwargs, timeName, calunits, caltype, times=csml.API.genSubset.genericSubset(self, csmlpath, ncpath, self.domain, kwargs) 
    4747       
    4848    #Find the original time name (timeAxis) for the file. 
    … …  
    5454               
    5555 
    56    #get the ratio of times to files 
    57     timeToFileRatio = csml.API.genSubset._getTimeToFileRatio(self, self.domain, timeName) 
    58          
    59              
    60     #list to keep track of files that have already been fetched. eg. if multiple times are in a single file only need to get data from that file once... 
    61     filesFetched=[] 
     56 
    6257         
    6358    ##Get names of variables in file and relate them to the subset selection  
    … …  
    6762     
    6863     
    69     self.files=[] 
    70     strTimes='' 
    71     fulldata=[] 
     64    strTimes, axisorder, fulldata=csml.API.genSubset.getTheData(self, selection, times, timeName) 
    7265     
    7366     
    74     for time in times: 
    75         listPosition=self.domain[timeName].index(time) 
    76         strTimes= strTimes + ' ' + time 
    77         for comp in csmlutils.listify(self.value.rangeSet.valueArray.valueComponent.quantityList.__insertedExtract.components):  
    78             filePos=int(float(listPosition)/timeToFileRatio) 
    79             if filePos in filesFetched: 
    80                 continue #already got data from this file, try next time 
    81             print selection 
    82             data=comp.getData(fileposition=filePos, **selection) 
    83             print data 
    84              
    85                  
    86             self.files.append(comp.fileList.fileNames.CONTENT[filePos]) #TODO, get the right file name 
    87             if fulldata ==[]: 
    88                 fulldata = data.tolist() 
    89                 print fulldata 
    90             else: 
    91                 for item in data.tolist(): 
    92                     fulldata.append(item) 
    93             filesFetched.append(filePos) 
    94         axisorder = data.getAxisIds()  #will need later! 
    95  
    96     #elif hasattr(self.value.rangeSet, 'datablock'): #not tested 
    97         #pass 
     67    #Okay, got the data now. Need to write CSML feature and NetCDF files. 
     68    #Writing out the CSML feature 
    9869     
    99     #Okay, got the data now. Need to write out CSML and NetCDF files... 
    100   
    101  
    102     #Writing out the CSML file 
    10370    # define domain/coverage  to use in 'value' attribute    
    104      
    10571    domain=csml.parser.GridSeriesDomain() 
    106     grid=csml.parser.GridCoordinatesTable() 
    107     domainSubset, totalArraySize=csml.API.genSubset._subsetDomain(timeName,strTimes,self.domain, **kwargs) 
    108     cTT=csml.parser.GridCoordinatesTable() 
    109     ords =[] 
    110     for key in domainSubset.keys(): 
    111         go=csml.parser.GridOrdinateDescription() 
    112         go.coordAxisLabel=csml.parser.csString(key) 
    113         go.gridAxesSpanned=csml.parser.csString(key) 
    114         go.coordAxisValues = csml.parser.SpatialOrTemporalPositionList() 
    115         if key==crs.axes[crs.timeAxis]: 
    116             go.coordAxisValues.timePositionList=csml.parser.csString(domainSubset[key]) #self.domain.key placeholder 
    117         else: 
    118             go.coordAxisValues.coordinateList=csml.parser.csString(domainSubset[key]) #self.domain.key placeholder 
    119         ords.append(go) #go needs a few more properties setting 
    120     cTT.gridOrdinates=ords 
    121     grid=self.value.gridSeriesDomain.coordTransformTable 
     72    domainSubset, totalArraySize=csml.API.genSubset.subsetDomain(timeName,strTimes,self.domain, **kwargs) 
     73    cTT=csml.API.genSubset.getCoordTransformTable(domainSubset, crs) 
    12274    domain.coordTransformTable=cTT 
    12375    rangeSet=csml.parser.RangeSet() 
    12476    rangeSet.arrayDescriptor=csml.parser.NetCDFExtract(id=self.id,fileName=csml.parser.csString(pathToSubsetNetCDF),variableName=csml.parser.csString(self.id),arraySize=csml.parser.csString(totalArraySize)) 
     77     
     78    #gridseries coverage 
    12579    cvg=csml.parser.GridSeriesCoverage() 
    12680    cvg.rangeSet=rangeSet 
    12781    cvg.gridSeriesDomain=domain     
    12882    csmlWrap=csml.csmllibs.csmlfeaturewrap.CSMLWrapper() 
     83     
     84    #create a stand alone gridseries feature containing this coverage 
    12985    subsettedFeature=csmlWrap.createGridSeriesFeature(value=cvg,datasetID="A",featureID="B",description="C") 
    130     
     86  
    13187     
    13288    ### write netcdf using NCWriter class (wraps cdms) ### 
    13389    nc=csml.csmllibs.netCDFWriter.NCwriter(pathToSubsetNetCDF) 
    134     floatTimes=[] 
    135     for time in times: 
    136         time=csml.API.ops_AbstractFeature.__getCDtime(time).torel(calunits) 
    137         floatTimes.append(time.value) 
    138     nc.addAxis('time',floatTimes,isTime=1,units=calunits,calendar=caltype) 
     90    ords=cTT.gridOrdinates 
     91    nc.genWriteVar(self.id,ords, times, calunits, caltype, axisorder, fulldata) 
     92    nc.closeFinishedFile() 
    13993     
    140     for ord in ords: 
    141         vals=[] 
    142         lon,lat=None,None 
    143         if ord.coordAxisLabel.CONTENT=='time': 
    144             continue 
    145         else: 
    146             for val in ord.coordAxisValues.coordinateList.CONTENT.split(','): 
    147                 if val != ' ': 
    148                     vals.append(float(val))   
    149         if ord.coordAxisLabel.CONTENT=='longitude': 
    150             lon=1 
    151             name='longitude' 
    152         elif ord.coordAxisLabel.CONTENT=='latitude': 
    153             lat=1 
    154             name='latitude' 
    155         else: 
    156             name=ord.coordAxisLabel.CONTENT 
    157   
    158         nc.addAxis(name,vals,isLon=lon,isLat=lat,units='')#to do, units attribute for CF compliance 
    159     if len(ords)==3: 
    160         axes=['time',axisorder[1],axisorder[2]] 
    161     elif len(ords)==2: 
    162         axes=['time',axisorder[1]] 
    163     nc.addVariable(fulldata,self.id, axes,units='') #to do, units attribute for CF compliance 
    164     nc.closeFinishedFile() 
    16594    return subsettedFeature, pathToSubsetNetCDF 
    16695     

  • TI02-CSML/trunk/csml/csmllibs/csmltime.py

    r2060 r2095  
    3838    elif cal =='julian': 
    3939        cdtime.DefaultCalendar=  cdtime.JulianCalendar 
     40         
     41         
     42def identifyTimeAxis(self,listToCheck):  
     43    #given a list of possible axes, find the one which is most probably the time axis. 
     44    for name in listToCheck: 
     45        if string.upper(name) in ['T', 'TIME']: 
     46            tname =name 
     47    return tname 
     48 
     49def compareTimes(timemin, timecheck, timemax): 
     50    timeok=0 
     51    try: 
     52        year=int(timecheck.split('-')[0]) 
     53        month=int(timecheck.split('-')[1]) 
     54        day=int(timecheck.split('-')[2].split('T')[0]) 
     55        tcheck=cdtime.comptime(year,month,day) 
     56        timeok=0 
     57        if tcheck.cmp(timemin) >= 0: 
     58            if tcheck.cmp(timemax) < 1: 
     59                timeok = 1 
     60        print 'TMIN time  %s'%timemin 
     61        print 'CHECK time  %s'%tcheck 
     62        print 'TMAX time  %s'%timemax 
     63    except: 
     64        #if it's not a proper internet time e.g. time = '6' then just compare numerically 
     65        timeok=0 
     66        if timecheck >=timemin: 
     67            if timecheck <=timemax: 
     68                timeok = 1 
     69    return timeok 
     70 
     71def getCDtime(time):  
     72    year=int(time.split('-')[0]) 
     73    month=int(time.split('-')[1]) 
     74    day=int(time.split('-')[2].split('T')[0]) 
     75    hour=int(time.split('-')[2].split('T')[1].split(':')[0]) 
     76    minute=int(time.split('-')[2].split('T')[1].split(':')[1]) 
     77    second=float(time.split('-')[2].split('T')[1].split(':')[2]) 
     78    cdTimeVal=cdtime.comptime(year,month,day,hour,minute,second) 
     79    return cdTimeVal 
    4080 
    4181def getFileTimeList(filelist, timedim): 

  • TI02-CSML/trunk/csml/csmllibs/netCDFWriter.py

    r2078 r2095  
    11import cdms,MV 
     2import csml 
    23import sys 
     4 
     5 
     6 
     7 
     8 
    39class NCwriter(object): 
    410    #This provides a simplified wrapper to CDMS to write a CF compliant NetCDF 
    … …  
    4652        self.nc.write(dataarray) 
    4753 
    48      
     54    def genWriteVar(self,varid, ordinates, times, calunits, caltype, axisorder, fulldata): 
     55        floatTimes=[] 
     56        for time in times: 
     57            time=csml.csmllibs.csmltime.getCDtime(time).torel(calunits) 
     58            floatTimes.append(time.value) 
     59        self.addAxis('time',floatTimes,isTime=1,units=calunits,calendar=caltype) 
     60        for ord in ordinates: 
     61            vals=[] 
     62            lon,lat=None,None 
     63            if ord.coordAxisLabel.CONTENT=='time': 
     64                continue 
     65            else: 
     66                for val in ord.coordAxisValues.coordinateList.CONTENT.split(','): 
     67                    if val != ' ': 
     68                        vals.append(float(val))   
     69            if ord.coordAxisLabel.CONTENT=='longitude': 
     70                lon=1 
     71                name='longitude' 
     72            elif ord.coordAxisLabel.CONTENT=='latitude': 
     73                lat=1 
     74                name='latitude' 
     75            else: 
     76                name=ord.coordAxisLabel.CONTENT 
     77            self.addAxis(name,vals,isLon=lon,isLat=lat,units='')#to do, units attribute for CF compliance 
     78        if len(ordinates)==3: 
     79            axes=['time',axisorder[1],axisorder[2]] 
     80        elif len(ordinates)==2: 
     81            axes=['time',axisorder[1]] 
     82        self.addVariable(fulldata,varid, axes,units='') #to do, units attribute for CF compliance 
     83                 
     84         
    4985    def closeFinishedFile(self): 
    5086        #returns finished (hopefully) NetCDF file 
Note: See TracChangeset for help on using the changeset viewer.