Changeset 3282


Ignore:
Timestamp:
29/01/08 11:59:44 (11 years ago)
Author:
domlowe
Message:

Feature file maps now operate on a directory tree instance rather than a path. Enables construction of directory trees from arbitrary collections of files

Location:
TI02-CSML/trunk/csml
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • TI02-CSML/trunk/csml/csmllibs/csmlbuilder.py

    r3277 r3282  
    55    '''this class contains all the method calls to create and populate a csml document. Calling the build method will start the build process 
    66    it needs to be provided with several bits of information to be able to do this. Csmlscan uses this class to run the build process.''' 
    7     def __init__(self,datasetid, directory,csmlft, mapping, timedimension ,outputfile,printscreen,timestorage,spatialstorage,valuestorage, minaxes, secallow, secdeny): 
     7    def __init__(self,datasetid, directory,selection,csmlft, mapping, timedimension ,outputfile,printscreen,timestorage,spatialstorage,valuestorage, minaxes, secallow, secdeny): 
    88        ''' The entry point for building a csml document.  
    99        @param datasetid:   identifier for the Dataset id attributes    
    1010        @param directory:   directory in which to start scanning 
     11        @param selection:   individual files paths to scan (do not need this and a directory) 
    1112        @param csmlft:    name of the feature type 
    1213        @param mapping:     name of the feature to file mapping (e.g. 'onetomany') 
     
    2829            self.datasetid =csml.csmllibs.csmlextra.getRandomID() 
    2930        self.directory = directory # top level directory containing data files  
     31        self.selection=selection 
    3032        self.csmlfeaturetype  = csmlft  
    3133        self.mapping = mapping  
     
    125127        ''' creates the  FeatureFileMap''' 
    126128        print "Create FFMAP" 
    127         fmm = csml.csmllibs.csmlfiles.FileMapMaker(self.directory, self.csmlfeaturetype) 
     129        if self.selection is not None: 
     130            #create a 'virtual' directory object that behaves like a normal directory. 
     131            directorytree=csml.csmllibs.csmldirectory.VirtualDirectoryTree(self.selection) 
     132        else: 
     133            directorytree=csml.csmllibs.csmldirectory.DirectoryTree(self.directory) 
     134        fmm = csml.csmllibs.csmlfiles.FileMapMaker(directorytree, self.csmlfeaturetype) 
    128135        if self.mapping=='onetomany': 
    129136            self.ffmap = fmm.onetomany() 
  • TI02-CSML/trunk/csml/csmllibs/csmldirectory.py

    r3279 r3282  
    1212class DirectoryTree(object): 
    1313        ''' Represents a directory''' 
    14         def __init__(self): 
    15                 self.subdirlist=[] 
     14        def __init__(self, topdir): 
     15            ''' sets the topdir attribute and reads the directory''' 
     16            self.subdirlist=[] 
     17            self.topdir=topdir 
     18            self._readDirectory() 
    1619                 
    17         def setTopDirectory(self,top): 
    18                 self.topdir=top 
    1920 
    2021        def readSubDir(self, subdir):    
     
    3435 
    3536                 
    36         def readDirectory(self): 
     37        def _readDirectory(self): 
    3738                ''' read directory and add listing of files to subdirlist''' 
    3839                self.subdirlist.append(self.topdir) 
     
    106107                 
    107108                         
    108                          
     109#class VirtualDirectoryTree(object): 
     110    #'''  TODO:Given a list of files this class has the same interface as a DirectoryTree'''                     
    109111                         
    110112                  
  • TI02-CSML/trunk/csml/csmllibs/csmlfiles.py

    r2222 r3282  
    1111    to build a featurefilemap object of featuretypes/files. 
    1212    current relationships are onetomany, onetoone and mixedft, 
    13     but others could be created ''' 
    14     def __init__(self,topdirectory, ftorftdictionary): 
    15             self.topdir=topdirectory 
     13    but others could be created. ''' 
     14    def __init__(self,directorytree, ftorftdictionary): 
     15            self.tree = directorytree 
    1616            if type(ftorftdictionary) is str: 
    1717                    self.ft = ftorftdictionary 
     
    2525    def onetomany(self): 
    2626        ''' onetomany means one file per directory/subdirectory is the representative file ''' 
    27         tree = csml.csmllibs.csmldirectory.DirectoryTree() 
    28         tree.setTopDirectory(self.topdir) 
    29         tree.readDirectory() 
    30         subdirs = tree.getSubDirectories() 
     27        subdirs = self.tree.getSubDirectories() 
    3128        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() 
    3229        for dir in subdirs: 
    33                 file=tree.getFirstInSubDir(dir) 
     30                file=self.tree.getFirstInSubDir(dir) 
    3431                if file == None: 
    3532                        continue 
     
    3835                repfile.setFeatureType(self.ft) 
    3936                try: 
    40                         otherfiles=tree.getAllCSMLFilesExceptFirst(dir) 
     37                        otherfiles=self.tree.getAllCSMLFilesExceptFirst(dir) 
    4138                        errtest=otherfiles[0] 
    4239                except: 
     
    5855    def onetoseveral(self): 
    5956        ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually. ''' 
    60         tree = csml.csmllibs.csmldirectory.DirectoryTree() 
    61         tree.setTopDirectory(self.topdir) 
    62         tree.readDirectory() 
    63         subdirs = tree.getSubDirectories() 
     57        subdirs = self.tree.getSubDirectories() 
    6458        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() 
    6559         
     
    136130 
    137131        for dir in subdirs: 
    138             file=tree.getFirstInSubDir(dir) 
     132            file=self.tree.getFirstInSubDir(dir) 
    139133            if file == None: 
    140134                continue 
     
    143137            repfile.setFeatureType(self.ft) 
    144138            try: 
    145                     otherfiles=tree.getAllCSMLFilesExceptFirst(dir) 
     139                    otherfiles=self.tree.getAllCSMLFilesExceptFirst(dir) 
    146140                    errtest=otherfiles[0] 
    147141            except: 
     
    162156        """ onetoone means each feature is self contained within any individual file 
    163157        """ 
    164         tree = csml.csmllibs.csmldirectory.DirectoryTree() 
    165         tree.setTopDirectory(self.topdir) 
    166         tree.readDirectory() 
    167         subdirs = tree.getSubDirectories() 
     158        subdirs = self.tree.getSubDirectories() 
    168159        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() 
    169160        for dir in subdirs:      
    170                 files=tree.getCSMLSupportedFilesInSubDir(dir) 
     161                files=self.tree.getCSMLSupportedFilesInSubDir(dir) 
    171162                for f in files: 
    172163                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f) 
     
    179170        assumes no file in toplevel directory, and then lots of subdirectories at next level containing files 
    180171        """ 
    181         tree = csml.csmllibs.csmldirectory.DirectoryTree() 
    182         tree.setTopDirectory(self.topdir) 
    183         tree.readDirectory() 
    184         subdirs = tree.getSubDirectories() 
    185         allfiles= tree.getAllCSMLSupportedFiles() 
     172        subdirs = self.tree.getSubDirectories() 
     173        allfiles= self.tree.getAllCSMLSupportedFiles() 
    186174        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() 
    187175         
     
    197185        return ffmap 
    198186 
    199  
    200              
     187            
    201188                         
    202189    def mixedft(self): 
    203190        ''' allows the use of a dictonary object to store feature file map info not well tested ''' 
    204         tree = csml.csmllibs.csmldirectory.DirectoryTree() 
    205         tree.setTopDirectory(self.topdir) 
    206         tree.readDirectory() 
    207         subdirs = tree.getSubDirectories() 
     191        subdirs = self.tree.getSubDirectories() 
    208192        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() 
    209193        for dir in subdirs:      
    210                 files=tree.getCSMLSupportedFilesInSubDir(dir) 
     194                files=self.tree.getCSMLSupportedFilesInSubDir(dir) 
    211195                for f in files: 
    212196                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f) 
  • TI02-CSML/trunk/csml/csmlscan.py

    r3262 r3282  
    2626    Scan a list of files (netCDF/PP/GRIB) producing a CSML file for those files  
    2727    (as a single CSML Dataset) 
     28    NOTE: these arguments are out of date. 
    2829 
    2930Arguments: 
     
    131132            MINAXES=0 
    132133        ROOTDIRECTORY = config.getConfigNoError('files', 'root') 
     134        SELECTION=config.getConfigNoError('files', 'selection') 
    133135        MAPPING = config.getConfigNoError('files', 'mapping') 
    134136        OUTPUTFILE = config.getConfigNoError('files', 'output') 
     
    207209    #build CSML document 
    208210    print ROOTDIRECTORY 
    209     csmldataset=csmllibs.csmlbuilder.csmlBuilder(DATASETID,ROOTDIRECTORY,FEATURETYPE,MAPPING,TIMEDIMENSION, OUTPUTFILE, PRINTSCREEN,TIMESTORAGE,SPATIALSTORAGE,VALUESTORAGE, MINAXES, SEC_ALLOW, SEC_DENY) 
     211    csmldataset=csmllibs.csmlbuilder.csmlBuilder(DATASETID,ROOTDIRECTORY,SELECTION,FEATURETYPE,MAPPING,TIMEDIMENSION, OUTPUTFILE, PRINTSCREEN,TIMESTORAGE,SPATIALSTORAGE,VALUESTORAGE, MINAXES, SEC_ALLOW, SEC_DENY) 
    210212    csmldataset.build() 
    211213 
Note: See TracChangeset for help on using the changeset viewer.