source: TI02-CSML/trunk/csml/csmllibs/csmlfiles.py @ 2126

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI02-CSML/trunk/csml/csmllibs/csmlfiles.py@2126
Revision 2126, 9.6 KB checked in by domlowe, 13 years ago (diff)

changing paths in config files

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#version 0.1
2#Dominic Lowe, BADC 8th December 2005
3#import csmllibs
4import sys
5import csml.csmllibs.csmldirectory
6import csml.csmllibs.csmldataiface
7
8
9class FileMapMaker(object):
10    ''' class which handles the lower level featurefilemap and directorytree classes
11    to build a featurefilemap object of featuretypes/files.
12    current relationships are onetomany, onetoone and mixedft,
13    but others could be created '''
14    def __init__(self,topdirectory, ftorftdictionary):
15            self.topdir=topdirectory
16            if type(ftorftdictionary) is str:
17                    self.ft = ftorftdictionary
18            elif type(ftorftdictionary) is dict:
19                    self.ftdictionary=ftorftdictionary
20            else:
21                    #wrong type
22                    print 'FileMapMaker takes a  featuretype string or dictionary'
23                    sys.exit()
24   
25    def onetomany(self):
26        ''' onetomany means one file per directory/subdirectory is the representative file '''
27        tree = csml.csmllibs.csmldirectory.DirectoryTree()
28        tree.setTopDirectory(self.topdir)
29        tree.readDirectory()
30        subdirs = tree.getSubDirectories()
31        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
32        for dir in subdirs:
33                file=tree.getFirstInSubDir(dir)
34                if file == None:
35                        continue
36                #create representative file
37                repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
38                repfile.setFeatureType(self.ft)
39                try:
40                        otherfiles=tree.getAllCSMLFilesExceptFirst(dir)
41                        errtest=otherfiles[0]
42                except:
43                        print 'no other csml files in this directory'
44                        #add representative file with no related files.
45                        ffmap.addRepresentativeFile(repfile)
46                        continue
47               
48                #add otherfiles as related files with feature type, ft
49                #note they could have individual featuretypes.
50
51                for f in otherfiles:
52                        relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
53                        relfile.setFeatureType(self.ft)
54                        repfile.addRelatedFile(relfile)
55                ffmap.addRepresentativeFile(repfile)
56        return ffmap
57
58    def onetoseveral(self):
59        ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually. '''
60        tree = csml.csmllibs.csmldirectory.DirectoryTree()
61        tree.setTopDirectory(self.topdir)
62        tree.readDirectory()
63        subdirs = tree.getSubDirectories()
64        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
65       
66        def __lookForSame(repFile, otherfiles):
67            '''recursively looks in list of otherfiles for files that match file in terms of dimensions (name + shape) and variables)'''
68            #this section compares the dimensions (name and shape) and variables in the other files with this representative file
69            repShapes=[]
70            DI=csml.csmllibs.csmldataiface.DataInterface()
71            DI=DI.getUnknownInterfaceType(repFile.getRepresentativeFileName())
72            DI.openFile(repFile.getRepresentativeFileName())
73            repVars=DI.getListofVariables()
74            tmpDims=DI.getListOfAxes()
75            repDims=[]
76            for dim in tmpDims:
77                if dim[:5] == 'axis_':  #this is just to ignore the autogenerated axis_n dimensions used by cdms...
78                    pass
79                else:
80                    repDims.append(dim)
81            for dim in repDims:
82                DI.setAxis(dim)
83                print DI.file
84                repShapes.append(DI.getSizeOfAxis(dim))
85                print dim
86                print DI.getSizeOfAxis(dim)
87            DI.closeFile()
88            nextotherfiles=[]
89            for otherfile in otherfiles:
90                compareShapes=[]
91                DI=csml.csmllibs.csmldataiface.DataInterface()
92                DI=DI.getUnknownInterfaceType(otherfile)
93                DI.openFile(otherfile)
94                compareVars=DI.getListofVariables()
95                tmpDims=DI.getListOfAxes()
96                compareDims=[]
97                for dim in tmpDims:
98                    if dim[:5] == 'axis_':  #again, skip the auto axis as they aren't a reliable measure
99                        pass
100                    else:
101                        compareDims.append(dim)
102                for dim in compareDims:
103                    DI.setAxis(dim)
104                    compareShapes.append(DI.getSizeOfAxis(dim))
105                if compareShapes + compareVars + compareDims == repShapes + repVars + repDims:
106                    #found a match, add relatedFile to representativeFile
107                    #print 'match between %s and %s'%(repFile.getRepresentativeFileName(), otherfile)
108                    #print file
109                    #print compareShapes + compareVars + compareDims
110                    #print otherfile
111                    #print repShapes + repVars + repDims
112                    relFile = csml.csmllibs.csmlfeaturefilemap.relatedFile(otherfile)
113                    relFile.setFeatureType(self.ft)
114                    repFile.addRelatedFile(relFile)
115                else:
116                    #keep this file for the next check
117                    nextotherfiles.append(otherfile)  #can't delete from list and loop at same time so use new list.
118                DI.closeFile()
119                otherfiles=nextotherfiles
120            ffmap.addRepresentativeFile(repFile)
121            if otherfiles !=[]:
122                nextfile=otherfiles[0]
123                nextrepFile=csml.csmllibs.csmlfeaturefilemap.representativeFile(nextfile)
124                nextrepFile.setFeatureType(self.ft)
125                otherfiles=__lookForSame(nextrepFile, otherfiles)
126            return otherfiles
127       
128
129        for dir in subdirs:
130            file=tree.getFirstInSubDir(dir)
131            if file == None:
132                continue
133            #create first representative file
134            repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
135            repfile.setFeatureType(self.ft)
136            try:
137                    otherfiles=tree.getAllCSMLFilesExceptFirst(dir)
138                    errtest=otherfiles[0]
139            except:
140                    print 'no other csml files in this directory'
141                    ffmap.addRepresentativeFile(repfile)
142                    continue
143            otherfiles=__lookForSame(repfile, otherfiles)
144            #for repf in ffmap.getRepresentativeFiles():
145                #print '******************'
146                #print 'representative:'
147                #print repf.getRepresentativeFileName()
148                #print 'related:'
149                #for relf in repf.getRelatedFiles():
150                    #print relf.getRelatedFileName()
151        return ffmap
152               
153    def onetoone(self):
154        """ onetoone means each feature is self contained within any individual file
155        """
156        tree = csml.csmllibs.csmldirectory.DirectoryTree()
157        tree.setTopDirectory(self.topdir)
158        tree.readDirectory()
159        subdirs = tree.getSubDirectories()
160        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
161        for dir in subdirs:     
162                files=tree.getCSMLSupportedFilesInSubDir(dir)
163                for f in files:
164                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
165                        repfile.setFeatureType(self.ft)
166                        ffmap.addRepresentativeFile(repfile)
167        return ffmap   
168       
169    def oneonly(self):
170        """ oneonly means one file represents feature spanning multiple directories
171        assumes no file in toplevel directory, and then lots of subdirectories at next level containing files
172        """
173        tree = csml.csmllibs.csmldirectory.DirectoryTree()
174        tree.setTopDirectory(self.topdir)
175        tree.readDirectory()
176        subdirs = tree.getSubDirectories()
177        allfiles= tree.getAllCSMLSupportedFiles()
178        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
179       
180        #create representative file from first file
181        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(allfiles[0])
182        repfile.setFeatureType(self.ft)
183        #create related files from all other files
184        for f in allfiles[1:]:
185            relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
186            relfile.setFeatureType(self.ft)
187            repfile.addRelatedFile(relfile)
188        ffmap.addRepresentativeFile(repfile)
189        return ffmap
190
191
192           
193                       
194    def mixedft(self):
195        ''' allows the use of a dictonary object to store feature file map info not well tested '''
196        tree = csml.csmllibs.csmldirectory.DirectoryTree()
197        tree.setTopDirectory(self.topdir)
198        tree.readDirectory()
199        subdirs = tree.getSubDirectories()
200        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
201        for dir in subdirs:     
202                files=tree.getCSMLSupportedFilesInSubDir(dir)
203                for f in files:
204                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
205                        #look up feature type in dictionary
206                        ft=self.ftdictionary[f]
207                        repfile.setFeatureType(ft)
208                        ffmap.addRepresentativeFile(repfile)   
209        return ffmap
Note: See TracBrowser for help on using the repository browser.