source: TI02-CSML/trunk/csml/csmllibs/csmlfiles.py @ 2122

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI02-CSML/trunk/csml/csmllibs/csmlfiles.py@2122
Revision 2122, 8.9 KB checked in by domlowe, 12 years ago (diff)

Last commit should have read onetoseveral not onetomany
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#version 0.1
2#Dominic Lowe, BADC 8th December 2005
3#import csmllibs
4import sys
5import csml.csmllibs.csmldirectory
6import csml.csmllibs.csmldataiface
7
8
9class FileMapMaker(object):
10    ''' class which handles the lower level featurefilemap and directorytree classes
11    to build a featurefilemap object of featuretypes/files.
12    current relationships are onetomany, onetoone and mixedft,
13    but others could be created '''
14    def __init__(self,topdirectory, ftorftdictionary):
15            self.topdir=topdirectory
16            if type(ftorftdictionary) is str:
17                    self.ft = ftorftdictionary
18            elif type(ftorftdictionary) is dict:
19                    self.ftdictionary=ftorftdictionary
20            else:
21                    #wrong type
22                    print 'FileMapMaker takes a  featuretype string or dictionary'
23                    sys.exit()
24   
25    def onetomany(self):
26        ''' onetomany means one file per directory/subdirectory is the representative file '''
27        tree = csml.csmllibs.csmldirectory.DirectoryTree()
28        tree.setTopDirectory(self.topdir)
29        tree.readDirectory()
30        subdirs = tree.getSubDirectories()
31        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
32        for dir in subdirs:
33                file=tree.getFirstInSubDir(dir)
34                if file == None:
35                        continue
36                #create representative file
37                repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
38                repfile.setFeatureType(self.ft)
39                try:
40                        otherfiles=tree.getAllCSMLFilesExceptFirst(dir)
41                        errtest=otherfiles[0]
42                except:
43                        print 'no other csml files in this directory'
44                        #add representative file with no related files.
45                        ffmap.addRepresentativeFile(repfile)
46                        continue
47               
48                #add otherfiles as related files with feature type, ft
49                #note they could have individual featuretypes.
50
51                for f in otherfiles:
52                        relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
53                        relfile.setFeatureType(self.ft)
54                        repfile.addRelatedFile(relfile)
55                ffmap.addRepresentativeFile(repfile)
56        return ffmap
57
58    def onetoseveral(self):
59        ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually. '''
60        tree = csml.csmllibs.csmldirectory.DirectoryTree()
61        tree.setTopDirectory(self.topdir)
62        tree.readDirectory()
63        subdirs = tree.getSubDirectories()
64        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
65       
66        def __lookForSame(repFile, otherfiles):
67            '''recursively looks in list of otherfiles for files that match file in terms of dimensions (name + shape) and variables)'''
68            #this section compares the dimensions (name and shape) and variables in the other files with this representative file
69            repShapes=[]
70            DI=csml.csmllibs.csmldataiface.DataInterface()
71            DI=DI.getUnknownInterfaceType(file)
72            DI.openFile(repFile.getRepresentativeFileName())
73            repVars=DI.getListofVariables()
74            tmpDims=DI.getListOfAxes()
75            repDims=[]
76            for dim in tmpDims:
77                if dim[:5] == 'axis_':  #this is just to ignore the autogenerated axis_n dimensions used by cdms...
78                    pass
79                else:
80                    repDims.append(dim)
81            for dim in repDims:
82                DI.setAxis(dim)
83                repShapes.append(DI.getSizeOfAxis(dim))
84            DI.closeFile()
85            nextotherfiles=[]
86            for otherfile in otherfiles:
87                compareShapes=[]
88                DI=csml.csmllibs.csmldataiface.DataInterface()
89                DI=DI.getUnknownInterfaceType(otherfile)
90                DI.openFile(otherfile)
91                compareVars=DI.getListofVariables()
92                tmpDims=DI.getListOfAxes()
93                compareDims=[]
94                for dim in tmpDims:
95                    if dim[:5] == 'axis_':  #again, skip the auto axis as they aren't a reliable measure
96                        pass
97                    else:
98                        compareDims.append(dim)
99                for dim in compareDims:
100                    DI.setAxis(dim)
101                    compareShapes.append(DI.getSizeOfAxis(dim))
102                if compareShapes + compareVars + compareDims == repShapes + repVars + repDims:
103                    #found a match, add relatedFile to representativeFile
104                    relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(otherfile)
105                    relfile.setFeatureType(self.ft)
106                    repFile.addRelatedFile(relfile)
107                else:
108                    #keep this file for the next check
109                    nextotherfiles.append(otherfile)  #can't delete from list and loop at same time so use new list.
110                DI.closeFile()
111                otherfiles=nextotherfiles
112            ffmap.addRepresentativeFile(repFile)
113            if otherfiles !=[]:
114                nextfile=otherfiles[0]
115                nextrepFile=csml.csmllibs.csmlfeaturefilemap.representativeFile(nextfile)
116                nextrepFile.setFeatureType(self.ft)
117                otherfiles=__lookForSame(nextrepFile, otherfiles)
118            return otherfiles
119       
120
121        for dir in subdirs:
122            file=tree.getFirstInSubDir(dir)
123            if file == None:
124                continue
125            #create first representative file
126            repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file)
127            repfile.setFeatureType(self.ft)
128            try:
129                    otherfiles=tree.getAllCSMLFilesExceptFirst(dir)
130                    errtest=otherfiles[0]
131            except:
132                    print 'no other csml files in this directory'
133                    ffmap.addRepresentativeFile(repfile)
134                    continue
135            otherfiles=__lookForSame(repfile, otherfiles)
136        return ffmap
137               
138    def onetoone(self):
139        """ onetoone means each feature is self contained within any individual file
140        """
141        tree = csml.csmllibs.csmldirectory.DirectoryTree()
142        tree.setTopDirectory(self.topdir)
143        tree.readDirectory()
144        subdirs = tree.getSubDirectories()
145        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
146        for dir in subdirs:     
147                files=tree.getCSMLSupportedFilesInSubDir(dir)
148                for f in files:
149                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
150                        repfile.setFeatureType(self.ft)
151                        ffmap.addRepresentativeFile(repfile)
152        return ffmap   
153       
154    def oneonly(self):
155        """ oneonly means one file represents feature spanning multiple directories
156        assumes no file in toplevel directory, and then lots of subdirectories at next level containing files
157        """
158        tree = csml.csmllibs.csmldirectory.DirectoryTree()
159        tree.setTopDirectory(self.topdir)
160        tree.readDirectory()
161        subdirs = tree.getSubDirectories()
162        allfiles= tree.getAllCSMLSupportedFiles()
163        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
164       
165        #create representative file from first file
166        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(allfiles[0])
167        repfile.setFeatureType(self.ft)
168        #create related files from all other files
169        for f in allfiles[1:]:
170            relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f)
171            relfile.setFeatureType(self.ft)
172            repfile.addRelatedFile(relfile)
173        ffmap.addRepresentativeFile(repfile)
174        return ffmap
175
176
177           
178                       
179    def mixedft(self):
180        ''' allows the use of a dictonary object to store feature file map info not well tested '''
181        tree = csml.csmllibs.csmldirectory.DirectoryTree()
182        tree.setTopDirectory(self.topdir)
183        tree.readDirectory()
184        subdirs = tree.getSubDirectories()
185        ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap()
186        for dir in subdirs:     
187                files=tree.getCSMLSupportedFilesInSubDir(dir)
188                for f in files:
189                        repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f)
190                        #look up feature type in dictionary
191                        ft=self.ftdictionary[f]
192                        repfile.setFeatureType(ft)
193                        ffmap.addRepresentativeFile(repfile)   
194        return ffmap
Note: See TracBrowser for help on using the repository browser.