source: TI02-CSML/trunk/csml/csmllibs/csmlfiles.py @ 3282

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI02-CSML/trunk/csml/csmllibs/csmlfiles.py@3282
Revision 3282, 9.3 KB checked in by domlowe, 13 years ago (diff)

Feature file maps now operate on a directory tree instance rather than a path. Enables construction of directory trees from arbitrary collections of files

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#version 0.1
2#Dominic Lowe, BADC 8th December 2005
3#import csmllibs
4import sys
5import csml.csmllibs.csmldirectory
6import csml.csmllibs.csmldataiface
7
8
9class FileMapMaker(object):
10    ''' class which handles the lower level featurefilemap and directorytree classes
11    to build a featurefilemap object of featuretypes/files.
12    current relationships are onetomany, onetoone and mixedft,
13    but others could be created. '''
14    def __init__(self,directorytree, ftorftdictionary):
15            self.tree = directorytree
16            if type(ftorftdictionary) is str:
17                    self.ft = ftorftdictionary
18            elif type(ftorftdictionary) is dict:
19                    self.ftdictionary=ftorftdictionary
20            else:
21                    #wrong type
22                    print 'FileMapMaker takes a  featuretype string or dictionary'
23                    sys.exit()
24   
25    def onetomany(self):
26        ''' onetomany means one file per directory/subdirectory is the representative file '''
27        subdirs = self.tree.getSubDirectories()
28        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
29        for dir in subdirs:
30                file=self.tree.getFirstInSubDir(dir)
31                if file == None:
32                        continue
33                #create representative file
34                repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
35                repfile.setFeatureType(self.ft)
36                try:
37                        otherfiles=self.tree.getAllCSMLFilesExceptFirst(dir)
38                        errtest=otherfiles[0]
39                except:
40                        print 'no other csml files in this directory'
41                        #add representative file with no related files.
42                        ffmap.addRepresentativeFile(repfile)
43                        continue
44               
45                #add otherfiles as related files with feature type, ft
46                #note they could have individual featuretypes.
47
48                for f in otherfiles:
49                        relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
50                        relfile.setFeatureType(self.ft)
51                        repfile.addRelatedFile(relfile)
52                ffmap.addRepresentativeFile(repfile)
53        return ffmap
54
55    def onetoseveral(self):
56        ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually. '''
57        subdirs = self.tree.getSubDirectories()
58        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
59       
60        def __lookForSame(repFile, otherfiles):
61            '''recursively looks in list of otherfiles for files that match file in terms of dimensions (name + shape) and variables)'''
62            #this section compares the dimensions (name and shape) and variables in the other files with this representative file
63            repShapes=[]
64            DI=csml.csmllibs.csmldataiface.DataInterface()
65            DI=DI.getUnknownInterfaceType(repFile.getRepresentativeFileName())
66            repName=repFile.getRepresentativeFileName()
67            DI.openFile(repName)
68            repVars=DI.getListofVariables()
69            tmpDims=DI.getListOfAxes()
70            repDims=[]
71            for dim in tmpDims:
72                if dim[:5] == 'axis_':  #this is just to ignore the autogenerated axis_n dimensions used by cdms...
73                    pass
74                else:
75                    repDims.append(dim)
76            for dim in repDims:
77                DI.setAxis(dim)
78                print DI.file
79                repShapes.append(DI.getSizeOfAxis(dim))
80                print dim
81                print DI.getSizeOfAxis(dim)
82            DI.closeFile()
83            nextotherfiles=[]
84            otherfilestemp=[]
85            for otherfile in otherfiles:
86                if otherfile == repName:
87                    pass
88                else:
89                    otherfilestemp.append(otherfile)
90            otherfiles = otherfilestemp
91            for otherfile in otherfiles:
92                compareShapes=[]
93                DI=csml.csmllibs.csmldataiface.DataInterface()
94                DI=DI.getUnknownInterfaceType(otherfile)
95                DI.openFile(otherfile)
96                compareVars=DI.getListofVariables()
97                tmpDims=DI.getListOfAxes()
98                compareDims=[]
99                for dim in tmpDims:
100                    if dim[:5] == 'axis_':  #again, skip the auto axis as they aren't a reliable measure
101                        pass
102                    else:
103                        compareDims.append(dim)
104                for dim in compareDims:
105                    DI.setAxis(dim)
106                    compareShapes.append(DI.getSizeOfAxis(dim))
107                if compareShapes + compareVars + compareDims == repShapes + repVars + repDims:
108                    #found a match, add relatedFile to representativeFile
109                    #print 'match between %s and %s'%(repFile.getRepresentativeFileName(), otherfile)
110                    #print file
111                    #print compareShapes + compareVars + compareDims
112                    #print otherfile
113                    #print repShapes + repVars + repDims
114                    relFile = csml.csmllibs.csmlfeaturefilemap.relatedFile(otherfile)
115                    relFile.setFeatureType(self.ft)
116                    repFile.addRelatedFile(relFile)
117                else:
118                    #keep this file for the next check
119                    nextotherfiles.append(otherfile)  #can't delete from list and loop at same time so use new list.
120                DI.closeFile()
121                otherfiles=nextotherfiles
122            ffmap.addRepresentativeFile(repFile)
123            if otherfiles !=[]:
124                nextfile=otherfiles[0]
125                nextrepFile=csml.csmllibs.csmlfeaturefilemap.representativeFile(nextfile)
126                nextrepFile.setFeatureType(self.ft)
127                otherfiles=__lookForSame(nextrepFile, otherfiles)
128            return otherfiles
129       
130
131        for dir in subdirs:
132            file=self.tree.getFirstInSubDir(dir)
133            if file == None:
134                continue
135            #create first representative file
136            repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
137            repfile.setFeatureType(self.ft)
138            try:
139                    otherfiles=self.tree.getAllCSMLFilesExceptFirst(dir)
140                    errtest=otherfiles[0]
141            except:
142                    print 'no other csml files in this directory'
143                    ffmap.addRepresentativeFile(repfile)
144                    continue
145            otherfiles=__lookForSame(repfile, otherfiles)
146            #for repf in ffmap.getRepresentativeFiles():
147                #print '******************'
148                #print 'representative:'
149                #print repf.getRepresentativeFileName()
150                #print 'related:'
151                #for relf in repf.getRelatedFiles():
152                    #print relf.getRelatedFileName()
153        return ffmap
154               
155    def onetoone(self):
156        """ onetoone means each feature is self contained within any individual file
157        """
158        subdirs = self.tree.getSubDirectories()
159        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
160        for dir in subdirs:     
161                files=self.tree.getCSMLSupportedFilesInSubDir(dir)
162                for f in files:
163                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
164                        repfile.setFeatureType(self.ft)
165                        ffmap.addRepresentativeFile(repfile)
166        return ffmap   
167       
168    def oneonly(self):
169        """ oneonly means one file represents feature spanning multiple directories
170        assumes no file in toplevel directory, and then lots of subdirectories at next level containing files
171        """
172        subdirs = self.tree.getSubDirectories()
173        allfiles= self.tree.getAllCSMLSupportedFiles()
174        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
175       
176        #create representative file from first file
177        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(allfiles[0])
178        repfile.setFeatureType(self.ft)
179        #create related files from all other files
180        for f in allfiles[1:]:
181            relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
182            relfile.setFeatureType(self.ft)
183            repfile.addRelatedFile(relfile)
184        ffmap.addRepresentativeFile(repfile)
185        return ffmap
186
187           
188                       
189    def mixedft(self):
190        ''' allows the use of a dictonary object to store feature file map info not well tested '''
191        subdirs = self.tree.getSubDirectories()
192        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
193        for dir in subdirs:     
194                files=self.tree.getCSMLSupportedFilesInSubDir(dir)
195                for f in files:
196                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
197                        #look up feature type in dictionary
198                        ft=self.ftdictionary[f]
199                        repfile.setFeatureType(ft)
200                        ffmap.addRepresentativeFile(repfile)   
201        return ffmap
Note: See TracBrowser for help on using the repository browser.