source: TI02-CSML/trunk/csml/API/ops_FileExtract.py @ 3267

''' ops_AbstractFileExtract  contains operations for AbstractFileExtracts which are inherited by any FileExtracts - eg NetCDFExtracts. Methods are attached on the fly.'''
import csml.csmllibs.csmldataiface
import sys
import Numeric
import MA


class DataContainer(object):
    ''' A container to wrap some of the inconsistencies of Numeric/MA etc
    Should convert to Numpy in future'''
    #dataArray needs to be the standards data container    
    def __init__(self, data=None):
        self.dataArray=None
        if data is not None:
            self.data=data
            self.setup()
    
    def setup(self):
        try: 
            self.dataArray=Numeric.array(self.data)
        except MA.MA.MAError:
            self.dataArray=self.data.tolist()
            self.dataArray=Numeric.array(self.dataArray)
          
def testmethod(self):
    '''test method'''
    print 'testmethod for AbstractFileExtract feature'
    return 'testmethod AbstractFileExtract'


def getAxisInfo(self, varName):
    pass

def getData(self,  fileposition=None,**kwargs):
    '''
    get data for a particular file in the file extract.
    @param fileposition:    position (optional) of file in filelist (of paths)
    @param kwargs:    a subset request dictionary
    @return:    dataArray, fillvalue, axisorder, units - array of data and ancilliary info.
    '''
    
    if csml.API.csmlContainer.globalContainer.OUTPUTDIR is not None:
        directory=csml.API.csmlContainer.globalContainer.OUTPUTDIR
    else:
        directory=None 
        
        
    #if fileposition is None, then this may be stored in the cache already.
    try:
        [dataArray, fillvalue, axisorder, units]=csml.API.csmlContainer.globalContainer.CACHE[self.id]
        return dataArray, fillvalue, axisorder, units
    except:
        pass
    
    #file position defines the position of the filename if a list of filenames exists
    #**kwargs can hold subsetting request.
    #get the right file  ##TO DO< check this is ok with FILE LIST
       
    # TODO - cannot subset times within an individual file - no way to determine the original time crs..
    
    if fileposition is not None:
        try:
            file =self.fileList.fileNames.CONTENT.split()[fileposition]
        except:
            file=self.fileName.CONTENT
    else:
        file=self.fileName.CONTENT
    #open the file
    DI = csml.csmllibs.csmldataiface.DataInterface()
    DI=DI.getUnknownInterfaceType(file)
    try:
        DI.openFile(file)
    except:
        if directory is not None:
            file=directory +'/' + file
            DI.openFile(file)
    DI.setAxis(self.variableName.CONTENT)
    try:
        DI.setAxis(self.variableName.CONTENT)
        data=DI.getDataForAxis()
        dataArray=data
        fillvalue=None
        axisorder=None
        units=None
    except:
        pass
        DI.setVariable(self.variableName.CONTENT)
        #get fill value (if there is one)   
        fillvalue=DI.getVariableAttribute('_FillValue')
        if kwargs:
            data = DI.getSubsetOfDataForVar(**kwargs)
        else:
            data = DI.getDataForVar()
        DC=DataContainer(data)
        dataArray=DC.dataArray
        units=[]
        axisorder=DI.getVariableAxes()
        for axis in axisorder:    
            DI.setAxis(axis)
            units.append(DI.getAxisAttribute('units'))
    DI.closeFile()
    
    #if fileposition is none, add to cache for retrieval by id later:
    try:
        csml.API.csmlContainer.globalContainer.CACHE[self.id]=[dataArray, fillvalue, axisorder, units]
    except:
        pass
    return dataArray, fillvalue, axisorder, units
        
def getDataFromChunks(self, minIdx, maxIdx):
    '''given a list of files of unknown length and an index range spanning some or all of those files, retuns the data from that index range. Only works with single dimensional data.
    e.g. used for QXF
    @param minIdx:   start index
    @param maxIndex:    end index
    @return:     dataarray, fillvalue    
    '''
    
    minIndex=minIdx
    maxIndex=maxIdx
    if csml.API.csmlContainer.globalContainer.OUTPUTDIR is not None:
        directory=csml.API.csmlContainer.globalContainer.OUTPUTDIR
    else:
        directory=None    
    data=[]
    totalarraylength = 0    
    filenames=[]
    try:
        for file in self.fileList.fileNames.CONTENT.split():
            filenames.append(file)
    except:
        filenames.append(self.fileName.CONTENT)
    
    for file in filenames:
        DI=csml.csmllibs.csmldataiface.DataInterface()
        DI=DI.getUnknownInterfaceType(file) 
        try:
            DI.openFile(file)
        except:
            if directory is not None:
                file=directory +'/' + file
                DI.openFile(file)
        DI.setVariable(self.variableName.CONTENT)
        datachunk=DI.getDataForVar()                   
        chunklength=len(datachunk)
        startpoint=totalarraylength
        totalarraylength=totalarraylength +chunklength
                
        if minIndex >= totalarraylength:
            #print 'before selection, skipping %s'%file
            continue
        
        elif minIndex < totalarraylength:
            if minIndex >= startpoint:
                sliceMin=minIndex-startpoint
                #print 'selection starts in %s at index %s'%(file, sliceMin)
                if maxIndex < totalarraylength:
                    #print 'selection self contained in %s'%file
                    sliceMax=sliceMin + (maxIndex-minIndex) +1
                    data=datachunk[sliceMin:sliceMax]
                    continue
                else:
                    #print 'selection goes beyond this file'
                    data.append(datachunk[sliceMin: len(datachunk)])
                    continue
        
        if maxIndex  < startpoint:
            #print 'gone past end of selection, skipping %s'%file
            continue
        elif maxIndex >= totalarraylength:
            #get the whole chunk
            data.append(datachunk)
            continue
        
        elif maxIndex <= totalarraylength:     
            sliceMax=(maxIndex-startpoint) +1
            data.append(datachunk[0:sliceMax])
            continue    
    fillvalue=DI.getVariableAttribute('_FillValue')
    axisorder=DI.getVariableAxes()
      
    DI.closeFile()
    if type(data) is MA.MA.MaskedArray:
        result =data
    else:
        result=MA.concatenate(data)
    #try:
        #result=MA.concatenate(data)
    #except:
        #result =data
    return result, fillvalue
        
def __calendar(file,timedim):
    ''' given a file and the name of the time dimension returns the name of the calendar and units used. eg 360day, days since 2001:01:01
    @param file:   filepath
    @param timedim:     name of time dimension
    @return:    calendar name, calendar units
    '''
    DI = csmldataiface.DataInterface()
    DI=DI.getUnknownInterfaceType(file)
    DI.openFile(file)
    DI.setAxis(timedim)
    cal=DI.getAxisAttribute('calendar')
    units=DI.getAxisAttribute('units')
    return cal, units