Changeset 2121 for TI02-CSML

Timestamp:

07/02/07 12:18:50 (14 years ago)

Author:

domlowe

Message:

New featurefilemap pattern onetomany, tested on DDC data

Location:

TI02-CSML/trunk/csml/csmllibs

Files:

• csmlbuilder.py (modified) (1 diff)
• csmldataiface.py (modified) (1 diff)
• csmlfiles.py (modified) (2 diffs)

TI02-CSML/trunk/csml/csmllibs/csmlbuilder.py

@@ -76 +76 @@
         if self.mapping=='onetomany':
             self.ffmap = fmm.onetomany()
+        elif self.mapping=='onetoseveral':
+            self.ffmap = fmm.onetoseveral()
         elif self.mapping=='onetoone':
             self.ffmap = fmm.onetoone()

TI02-CSML/trunk/csml/csmllibs/csmldataiface.py

@@ -44 +44 @@
                 #this function to examine the file and return the correct interface (if it exists).
                 fileExtension = str(filename)[-3:]
-                print 'fe: %s'%fileExtension
                 if fileExtension == '.nc':
                         return cdunifInterface()

TI02-CSML/trunk/csml/csmllibs/csmlfiles.py

@@ -4 +4 @@
 import sys
 import csml.csmllibs.csmldirectory
+import csml.csmllibs.csmldataiface
 
 
@@ -54 +55 @@
                 ffmap.addRepresentativeFile(repfile)
         return ffmap
-                
+
+    def onetoseveral(self):
+        ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually.'''
+        tree = csml.csmllibs.csmldirectory.DirectoryTree()
+        tree.setTopDirectory(self.topdir)
+        tree.readDirectory()
+        subdirs = tree.getSubDirectories()
+        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
+        
+        def __lookForSame(repFile, otherfiles):
+            '''recursively looks in list of otherfiles for files that match file in terms of dimensions (name + shape) and variables)'''
+            #this section compares the dimensions (name and shape) and variables in the other files with this representative file
+            repShapes=[]
+            DI=csml.csmllibs.csmldataiface.DataInterface()
+            DI=DI.getUnknownInterfaceType(file)
+            DI.openFile(repFile.getRepresentativeFileName())
+            repVars=DI.getListofVariables()
+            tmpDims=DI.getListOfAxes()
+            repDims=[]
+            for dim in tmpDims:
+                if dim[:5] == 'axis_':  #this is just to ignore the autogenerated axis_n dimensions used by cdms...
+                    pass
+                else:
+                    repDims.append(dim)
+            for dim in repDims:
+                DI.setAxis(dim)
+                repShapes.append(DI.getSizeOfAxis(dim))
+            DI.closeFile()
+            nextotherfiles=[]
+            for otherfile in otherfiles:
+                compareShapes=[]
+                DI=csml.csmllibs.csmldataiface.DataInterface()
+                DI=DI.getUnknownInterfaceType(otherfile)
+                DI.openFile(otherfile)
+                compareVars=DI.getListofVariables()
+                tmpDims=DI.getListOfAxes()
+                compareDims=[]
+                for dim in tmpDims:
+                    if dim[:5] == 'axis_':  #again, skip the auto axis as they aren't a reliable measure
+                        pass
+                    else:
+                        compareDims.append(dim)
+                for dim in compareDims:
+                    DI.setAxis(dim)
+                    compareShapes.append(DI.getSizeOfAxis(dim))
+                if compareShapes + compareVars + compareDims == repShapes + repVars + repDims:
+                    #found a match, add relatedFile to representativeFile
+                    relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(otherfile)
+                    relfile.setFeatureType(self.ft)
+                    repFile.addRelatedFile(relfile)
+                else:
+                    #keep this file for the next check
+                    nextotherfiles.append(otherfile)  #can't delete from list and loop at same time so use new list.
+                DI.closeFile()
+                otherfiles=nextotherfiles
+            ffmap.addRepresentativeFile(repFile)
+            if otherfiles !=[]:
+                nextfile=otherfiles[0]
+                nextrepFile=csml.csmllibs.csmlfeaturefilemap.representativeFile(nextfile)
+                nextrepFile.setFeatureType(self.ft)
+                otherfiles=__lookForSame(nextrepFile, otherfiles)
+            return otherfiles
+        
+
+        for dir in subdirs:
+            file=tree.getFirstInSubDir(dir)
+            if file == None:
+                continue
+            #create first representative file
+            repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
+            repfile.setFeatureType(self.ft)
+            try:
+                    otherfiles=tree.getAllCSMLFilesExceptFirst(dir)
+                    errtest=otherfiles[0]
+            except:
+                    print 'no other csml files in this directory'
+                    ffmap.addRepresentativeFile(repfile)
+                    continue
+            otherfiles=__lookForSame(repfile, otherfiles)
+        return ffmap
+                
     def onetoone(self):
         """ onetoone means each feature is self contained within any individual file