source: TI02-CSML/trunk/csml/csmllibs/csmlfiles.py @ 3286

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI02-CSML/trunk/csml/csmllibs/csmlfiles.py@3286
Revision 3286, 9.1 KB checked in by domlowe, 13 years ago (diff)

overhaul of scanning command args

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#version 0.1
2#Dominic Lowe, BADC 8th December 2005
3#import csmllibs
4import sys
5import csml.csmllibs.csmldirectory
6import csml.csmllibs.csmldataiface
7
8
9class FileMapMaker(object):
10    ''' class which handles the lower level featurefilemap and directorytree classes
11    to build a featurefilemap object of featuretypes/files.
12    current relationships are onetomany, onetoone and mixedft,
13    but others could be created. '''
14    def __init__(self,directorytree, ftorftdictionary):
15            self.tree = directorytree
16            if type(ftorftdictionary) is str:
17                    self.ft = ftorftdictionary
18            elif type(ftorftdictionary) is dict:
19                    self.ftdictionary=ftorftdictionary
20            else:
21                    #wrong type
22                    print 'FileMapMaker takes a  featuretype string or dictionary'
23                    sys.exit()
24   
25    def onetomany(self):
26        ''' onetomany means one file per directory/subdirectory is the representative file '''
27        subdirs = self.tree.getSubDirectories()
28        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
29        for dir in subdirs:
30                file=self.tree.getFirstInSubDir(dir)
31                if file == None:
32                        continue
33                #create representative file
34                repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
35                repfile.setFeatureType(self.ft)
36                try:
37                        otherfiles=self.tree.getAllCSMLFilesExceptFirst(dir)
38                        errtest=otherfiles[0]
39                except:
40                        #add representative file with no related files.
41                        ffmap.addRepresentativeFile(repfile)
42                        continue
43               
44                #add otherfiles as related files with feature type, ft
45                #note they could have individual featuretypes.
46
47                for f in otherfiles:
48                        relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
49                        relfile.setFeatureType(self.ft)
50                        repfile.addRelatedFile(relfile)
51                ffmap.addRepresentativeFile(repfile)
52        return ffmap
53
54    def onetoseveral(self):
55        ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually. '''
56        subdirs = self.tree.getSubDirectories()
57        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
58       
59        def __lookForSame(repFile, otherfiles):
60            '''recursively looks in list of otherfiles for files that match file in terms of dimensions (name + shape) and variables)'''
61            #this section compares the dimensions (name and shape) and variables in the other files with this representative file
62            repShapes=[]
63            DI=csml.csmllibs.csmldataiface.DataInterface()
64            DI=DI.getUnknownInterfaceType(repFile.getRepresentativeFileName())
65            repName=repFile.getRepresentativeFileName()
66            DI.openFile(repName)
67            repVars=DI.getListofVariables()
68            tmpDims=DI.getListOfAxes()
69            repDims=[]
70            for dim in tmpDims:
71                if dim[:5] == 'axis_':  #this is just to ignore the autogenerated axis_n dimensions used by cdms...
72                    pass
73                else:
74                    repDims.append(dim)
75            for dim in repDims:
76                DI.setAxis(dim)
77                repShapes.append(DI.getSizeOfAxis(dim))
78            DI.closeFile()
79            nextotherfiles=[]
80            otherfilestemp=[]
81            for otherfile in otherfiles:
82                if otherfile == repName:
83                    pass
84                else:
85                    otherfilestemp.append(otherfile)
86            otherfiles = otherfilestemp
87            for otherfile in otherfiles:
88                compareShapes=[]
89                DI=csml.csmllibs.csmldataiface.DataInterface()
90                DI=DI.getUnknownInterfaceType(otherfile)
91                DI.openFile(otherfile)
92                compareVars=DI.getListofVariables()
93                tmpDims=DI.getListOfAxes()
94                compareDims=[]
95                for dim in tmpDims:
96                    if dim[:5] == 'axis_':  #again, skip the auto axis as they aren't a reliable measure
97                        pass
98                    else:
99                        compareDims.append(dim)
100                for dim in compareDims:
101                    DI.setAxis(dim)
102                    compareShapes.append(DI.getSizeOfAxis(dim))
103                if compareShapes + compareVars + compareDims == repShapes + repVars + repDims:
104                    #found a match, add relatedFile to representativeFile
105                    #print 'match between %s and %s'%(repFile.getRepresentativeFileName(), otherfile)
106                    #print file
107                    #print compareShapes + compareVars + compareDims
108                    #print otherfile
109                    #print repShapes + repVars + repDims
110                    relFile = csml.csmllibs.csmlfeaturefilemap.relatedFile(otherfile)
111                    relFile.setFeatureType(self.ft)
112                    repFile.addRelatedFile(relFile)
113                else:
114                    #keep this file for the next check
115                    nextotherfiles.append(otherfile)  #can't delete from list and loop at same time so use new list.
116                DI.closeFile()
117                otherfiles=nextotherfiles
118            ffmap.addRepresentativeFile(repFile)
119            if otherfiles !=[]:
120                nextfile=otherfiles[0]
121                nextrepFile=csml.csmllibs.csmlfeaturefilemap.representativeFile(nextfile)
122                nextrepFile.setFeatureType(self.ft)
123                otherfiles=__lookForSame(nextrepFile, otherfiles)
124            return otherfiles
125       
126
127        for dir in subdirs:
128            file=self.tree.getFirstInSubDir(dir)
129            if file == None:
130                continue
131            #create first representative file
132            repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
133            repfile.setFeatureType(self.ft)
134            try:
135                    otherfiles=self.tree.getAllCSMLFilesExceptFirst(dir)
136                    errtest=otherfiles[0]
137            except:
138                    print 'no other csml files in this directory'
139                    ffmap.addRepresentativeFile(repfile)
140                    continue
141            otherfiles=__lookForSame(repfile, otherfiles)
142            #for repf in ffmap.getRepresentativeFiles():
143                #print '******************'
144                #print 'representative:'
145                #print repf.getRepresentativeFileName()
146                #print 'related:'
147                #for relf in repf.getRelatedFiles():
148                    #print relf.getRelatedFileName()
149        return ffmap
150               
151    def onetoone(self):
152        """ onetoone means each feature is self contained within any individual file
153        """
154        subdirs = self.tree.getSubDirectories()
155        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
156        for dir in subdirs:     
157                files=self.tree.getCSMLSupportedFilesInSubDir(dir)
158                for f in files:
159                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
160                        repfile.setFeatureType(self.ft)
161                        ffmap.addRepresentativeFile(repfile)
162        return ffmap   
163       
164    def oneonly(self):
165        """ oneonly means one file represents feature spanning multiple directories
166        assumes no file in toplevel directory, and then lots of subdirectories at next level containing files
167        """
168        subdirs = self.tree.getSubDirectories()
169        allfiles= self.tree.getAllCSMLSupportedFiles()
170        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
171       
172        #create representative file from first file
173        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(allfiles[0])
174        repfile.setFeatureType(self.ft)
175        #create related files from all other files
176        for f in allfiles[1:]:
177            relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
178            relfile.setFeatureType(self.ft)
179            repfile.addRelatedFile(relfile)
180        ffmap.addRepresentativeFile(repfile)
181        return ffmap
182
183           
184                       
185    def mixedft(self):
186        ''' allows the use of a dictonary object to store feature file map info not well tested '''
187        subdirs = self.tree.getSubDirectories()
188        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
189        for dir in subdirs:     
190                files=self.tree.getCSMLSupportedFilesInSubDir(dir)
191                for f in files:
192                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
193                        #look up feature type in dictionary
194                        ft=self.ftdictionary[f]
195                        repfile.setFeatureType(ft)
196                        ffmap.addRepresentativeFile(repfile)   
197        return ffmap
Note: See TracBrowser for help on using the repository browser.