Changes between Version 7 and Version 8 of CSMLReadMethods


Ignore:
Timestamp:
25/01/08 15:35:11 (12 years ago)
Author:
domlowe
Comment:

Added pml data interface example to documentation

Legend:

Unmodified
Added
Removed
Modified
  • CSMLReadMethods

    v7 v8  
    249249 
    250250There is a bit more to all this in terms of integrating it into the main trunk of the CSML code, but the first stage is to work on these methods for your file format. 
     251 
     252 
     253== Appendix 1: Data Interfaces for raw and image files == 
     254Staff at Plymouth Marine Laboratory have written !DataInterfaces to read image files and raw binary files. The following code shows how they did it: 
     255{{{ 
     256#!python 
     257class RawFileInterface(AbstractDI): 
     258 
     259   def __init__(self): 
     260      self.extractType   = 'rawExtract' 
     261      self.extractPrefix = '_rawextract_' 
     262  
     263  
     264   def openFile(self, filename): 
     265      self.file = open(filename, "rb") 
     266 
     267 
     268   def closeFile(self): 
     269      self.file.close() 
     270 
     271 
     272   # Read the data from the raw file into a multidimensional Numeric array 
     273   def readFile(self, **kwargs): 
     274        # Determine the numeric type: 
     275        if 'numericType' in kwargs: 
     276            try: 
     277                numericTypeCode = { 
     278                    'uint8':Numeric.UInt8, 
     279                    'uint16':Numeric.UInt16, 
     280                    'uint32':Numeric.UInt32, 
     281                    'int8':Numeric.Int8, 
     282                    'int16':Numeric.Int16, 
     283                    'int32':Numeric.Int32, 
     284                    'float':Numeric.Float32, 
     285                    'double':Numeric.Float64 
     286                }[kwargs['numericType']] 
     287            except KeyError: 
     288                raise TypeError("Invalid numericType: " + str(kwargs['numericType'])) 
     289        else: 
     290            raise KeyError("numericType is mandatory.") 
     291         
     292        # Read the file into a numpy array: 
     293        self.data = Numeric.fromstring(self.file.read(), numericTypeCode) 
     294        # If necessary, swap the byte order: 
     295        if 'endianess' in kwargs: 
     296            if ((kwargs['endianness'] == 'little' and not Numeric.LittleEndian) or (kwargs['endianness'] == 'big' and Numeric.LittleEndian)): 
     297                self.data = self.data.byteswapped()     
     298        # Declare the shape of the array: 
     299        dimensions = map(int,kwargs['dimensions']) 
     300        dimensions.reverse() 
     301        self.data.shape = tuple(dimensions) 
     302        # If numericTransform or fillValue were provided, store them as 
     303        # attributes. 
     304        if 'numericTransform' in kwargs: 
     305            self.numericTransform = NumericTransform.infixExpression(kwargs['numericTransform']) 
     306        if 'fillValue' in kwargs: 
     307            self.fillValue = kwargs['fillValue'] 
     308    
     309   # Return the fill value, if set, and transform if necessary: 
     310   def getFillValue(self): 
     311      # Both fillValue and numericTransform attributes may or may 
     312      # not exist... 
     313      try: 
     314         return self.numericTransform.solve( n = float(self.fillValue) ) 
     315      except AttributeError: 
     316         try: 
     317            return self.fillValue 
     318         except AttributeError: 
     319            return None 
     320 
     321 
     322   # This is a just a special case of getSubsetOfDataForVar. To avoid 
     323   # duplication of code, just subset the entire array. (getSubset.. is 
     324   # optimised for this case) 
     325   def getDataForVar(self): 
     326      return self.getSubsetOfDataForVar(lower = (0,)*len(self.data.shape), 
     327                                        upper = self.data.shape) 
     328 
     329 
     330   # Subset the n-dimensional file based on array indices. Accepts parameters 
     331   # in the form of e.g. 
     332   # 
     333   # getSubsetOfDataForVar( lower=(0,0), upper=(1,2) ) 
     334   # 
     335   # Note: The rank of the required subset, must exactly match the 
     336   # rank of the original data: len(lower) == len(upper) == rank of file 
     337   # 
     338   def getSubsetOfDataForVar(self, **kwargs): 
     339      # Assume subset parameters are passed as: lower=(0,0) upper=(512,512) 
     340      if 'lower' not in kwargs or 'upper' not in kwargs: 
     341         # Have not specified any subset parameters that we recognise, so raise 
     342         # an exception: 
     343         raise NotImplementedError("Only supports subsetting with lower+upper array indices") 
     344      elif not len(kwargs['lower']) == len(kwargs['upper']) == len(self.data.shape): 
     345         # Rank of subset is not the same as rank of full data array. so raise 
     346         # an exception: 
     347         raise NotImplementedError("Only supports subsets of same rank as full dataset") 
     348      elif Numeric.sometrue(Numeric.greater(kwargs['upper'], self.data.shape)): 
     349         # Requested upper bound of subset is beyond the size of the the full 
     350         # data array, so raise an exception 
     351         raise IndexError("Subset out of range") 
     352      elif Numeric.sometrue(Numeric.less( kwargs['upper'], Numeric.zeros(len(self.data.shape)))): 
     353         # Requested lower bound of subset is beyond the size of the the full 
     354         # data array, so raise an exception 
     355         raise IndexError("Subset out of range") 
     356      elif Numeric.sometrue(Numeric.less_equal(kwargs['upper'], kwargs['lower'])): 
     357         # lower bound <= upper_bound for at least one dimension, so raise an 
     358         # exception 
     359         raise IndexError("Upper bound less than lower bound") 
     360      elif tuple(kwargs['lower']) == (0,)*len(self.data.shape) and tuple(kwargs['upper']) == self.data.shape: 
     361         # Special case of requested subset == entire data file. 
     362         subset = self.data 
     363      else: 
     364         # We are okay to subset. 
     365 
     366         # I cant see any nice (and speedy) way of subsetting a Numeric 
     367         # array of arbitrary rank without the use of eval. By doing it 
     368         # this way, we can make use of the (possible) acceleration provided 
     369         # by Numeric/NumPy. 
     370         slices = [] 
     371         for i in range(len(self.data.shape)): 
     372            lower = int(kwargs['lower'][i]) 
     373            upper = int(kwargs['upper'][i])  
     374            slices.append(str(lower)+':'+str(upper)) 
     375         subset = eval("self.data["+','.join(slices)+"]") 
     376 
     377      # Attempt to perform the numericTransform on the data array, if we get 
     378      # AttributeError, it most likely means that numericTransform was not 
     379      # specified, so return the data as-is 
     380      try: 
     381         return self.numericTransform.solve( n = subset ) 
     382      except AttributeError: 
     383         return subset.copy() 
     384 
     385 
     386# Interface for reading data from image files. Requires PIL Image module. 
     387class ImageFileInterface(RawFileInterface): 
     388   def __init__(self): 
     389      self.extractType   = 'imageExtract' 
     390      self.extractPrefix = '_imageextract_' 
     391    
     392   def image2array(self,im): 
     393       #Adapted from code by Fredrik Lundh, http://www.pythonware.com  
     394       #http://effbot.org/zone/pil-numpy.htm 
     395        if im.mode not in ("L", "F"): 
     396            raise ValueError, "can only convert single-layer images" 
     397        if im.mode == "L": 
     398            a = Numeric.fromstring(im.tostring(), Numeric.UnsignedInt8) 
     399        else: 
     400            a = Numeric.fromstring(im.tostring(), Numeric.Float32) 
     401        a.shape = im.size[1], im.size[0] 
     402        return a 
     403 
     404   def openFile(self, filename): 
     405      self.file = Image.open(filename) 
     406 
     407   def closeFile(self): 
     408      self.file = None #...Image does not seem to have a close() method. 
     409 
     410   def readFile(self, **kwargs): 
     411      # Convert the image to a Numeric array 
     412       
     413      self.data=self.image2array(self.file) 
     414      #slower method: 
     415      #self.data = Numeric.array(self.file.getdata()) 
     416 
     417      if 'numericTransform' in kwargs: 
     418         # numericTransform was specified, so compile the expression: 
     419         self.numericTransform = NumericTransform.infixExpression(kwargs['numericTransform']) 
     420      if 'fillValue' in kwargs: 
     421         self.fillValue = kwargs['fillValue'] 
     422}}} 
     423 
     424 
     425 
     426