1 | #version 0.1 |
---|
2 | #Dominic Lowe, BADC 8th December 2005 |
---|
3 | #import csmllibs |
---|
4 | import sys |
---|
5 | import csml.csmllibs.csmldirectory |
---|
6 | import csml.csmllibs.csmldataiface |
---|
7 | |
---|
8 | |
---|
9 | class FileMapMaker(object): |
---|
10 | ''' class which handles the lower level featurefilemap and directorytree classes |
---|
11 | to build a featurefilemap object of featuretypes/files. |
---|
12 | current relationships are onetomany, onetoone and mixedft, |
---|
13 | but others could be created ''' |
---|
14 | def __init__(self,topdirectory, ftorftdictionary): |
---|
15 | self.topdir=topdirectory |
---|
16 | if type(ftorftdictionary) is str: |
---|
17 | self.ft = ftorftdictionary |
---|
18 | elif type(ftorftdictionary) is dict: |
---|
19 | self.ftdictionary=ftorftdictionary |
---|
20 | else: |
---|
21 | #wrong type |
---|
22 | print 'FileMapMaker takes a featuretype string or dictionary' |
---|
23 | sys.exit() |
---|
24 | |
---|
25 | def onetomany(self): |
---|
26 | ''' onetomany means one file per directory/subdirectory is the representative file ''' |
---|
27 | tree = csml.csmllibs.csmldirectory.DirectoryTree() |
---|
28 | tree.setTopDirectory(self.topdir) |
---|
29 | tree.readDirectory() |
---|
30 | subdirs = tree.getSubDirectories() |
---|
31 | ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() |
---|
32 | for dir in subdirs: |
---|
33 | file=tree.getFirstInSubDir(dir) |
---|
34 | if file == None: |
---|
35 | continue |
---|
36 | #create representative file |
---|
37 | repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file) |
---|
38 | repfile.setFeatureType(self.ft) |
---|
39 | try: |
---|
40 | otherfiles=tree.getAllCSMLFilesExceptFirst(dir) |
---|
41 | errtest=otherfiles[0] |
---|
42 | except: |
---|
43 | print 'no other csml files in this directory' |
---|
44 | #add representative file with no related files. |
---|
45 | ffmap.addRepresentativeFile(repfile) |
---|
46 | continue |
---|
47 | |
---|
48 | #add otherfiles as related files with feature type, ft |
---|
49 | #note they could have individual featuretypes. |
---|
50 | |
---|
51 | for f in otherfiles: |
---|
52 | relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f) |
---|
53 | relfile.setFeatureType(self.ft) |
---|
54 | repfile.addRelatedFile(relfile) |
---|
55 | ffmap.addRepresentativeFile(repfile) |
---|
56 | return ffmap |
---|
57 | |
---|
58 | def onetoseveral(self): |
---|
59 | ''' like one to many, but a directory may contain several representative files e.g. 5 files containing one (or more) feature(s), and then 5 files containing one (or more) different feature(s) etc. Need to examine the contents of each file to see if it's like another. Might be slow on large datsets as it examines the files individually. ''' |
---|
60 | tree = csml.csmllibs.csmldirectory.DirectoryTree() |
---|
61 | tree.setTopDirectory(self.topdir) |
---|
62 | tree.readDirectory() |
---|
63 | subdirs = tree.getSubDirectories() |
---|
64 | ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() |
---|
65 | |
---|
66 | def __lookForSame(repFile, otherfiles): |
---|
67 | '''recursively looks in list of otherfiles for files that match file in terms of dimensions (name + shape) and variables)''' |
---|
68 | #this section compares the dimensions (name and shape) and variables in the other files with this representative file |
---|
69 | repShapes=[] |
---|
70 | DI=csml.csmllibs.csmldataiface.DataInterface() |
---|
71 | DI=DI.getUnknownInterfaceType(file) |
---|
72 | DI.openFile(repFile.getRepresentativeFileName()) |
---|
73 | repVars=DI.getListofVariables() |
---|
74 | tmpDims=DI.getListOfAxes() |
---|
75 | repDims=[] |
---|
76 | for dim in tmpDims: |
---|
77 | if dim[:5] == 'axis_': #this is just to ignore the autogenerated axis_n dimensions used by cdms... |
---|
78 | pass |
---|
79 | else: |
---|
80 | repDims.append(dim) |
---|
81 | for dim in repDims: |
---|
82 | DI.setAxis(dim) |
---|
83 | repShapes.append(DI.getSizeOfAxis(dim)) |
---|
84 | DI.closeFile() |
---|
85 | nextotherfiles=[] |
---|
86 | for otherfile in otherfiles: |
---|
87 | compareShapes=[] |
---|
88 | DI=csml.csmllibs.csmldataiface.DataInterface() |
---|
89 | DI=DI.getUnknownInterfaceType(otherfile) |
---|
90 | DI.openFile(otherfile) |
---|
91 | compareVars=DI.getListofVariables() |
---|
92 | tmpDims=DI.getListOfAxes() |
---|
93 | compareDims=[] |
---|
94 | for dim in tmpDims: |
---|
95 | if dim[:5] == 'axis_': #again, skip the auto axis as they aren't a reliable measure |
---|
96 | pass |
---|
97 | else: |
---|
98 | compareDims.append(dim) |
---|
99 | for dim in compareDims: |
---|
100 | DI.setAxis(dim) |
---|
101 | compareShapes.append(DI.getSizeOfAxis(dim)) |
---|
102 | if compareShapes + compareVars + compareDims == repShapes + repVars + repDims: |
---|
103 | #found a match, add relatedFile to representativeFile |
---|
104 | relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(otherfile) |
---|
105 | relfile.setFeatureType(self.ft) |
---|
106 | repFile.addRelatedFile(relfile) |
---|
107 | else: |
---|
108 | #keep this file for the next check |
---|
109 | nextotherfiles.append(otherfile) #can't delete from list and loop at same time so use new list. |
---|
110 | DI.closeFile() |
---|
111 | otherfiles=nextotherfiles |
---|
112 | ffmap.addRepresentativeFile(repFile) |
---|
113 | if otherfiles !=[]: |
---|
114 | nextfile=otherfiles[0] |
---|
115 | nextrepFile=csml.csmllibs.csmlfeaturefilemap.representativeFile(nextfile) |
---|
116 | nextrepFile.setFeatureType(self.ft) |
---|
117 | otherfiles=__lookForSame(nextrepFile, otherfiles) |
---|
118 | return otherfiles |
---|
119 | |
---|
120 | |
---|
121 | for dir in subdirs: |
---|
122 | file=tree.getFirstInSubDir(dir) |
---|
123 | if file == None: |
---|
124 | continue |
---|
125 | #create first representative file |
---|
126 | repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(file) |
---|
127 | repfile.setFeatureType(self.ft) |
---|
128 | try: |
---|
129 | otherfiles=tree.getAllCSMLFilesExceptFirst(dir) |
---|
130 | errtest=otherfiles[0] |
---|
131 | except: |
---|
132 | print 'no other csml files in this directory' |
---|
133 | ffmap.addRepresentativeFile(repfile) |
---|
134 | continue |
---|
135 | otherfiles=__lookForSame(repfile, otherfiles) |
---|
136 | return ffmap |
---|
137 | |
---|
138 | def onetoone(self): |
---|
139 | """ onetoone means each feature is self contained within any individual file |
---|
140 | """ |
---|
141 | tree = csml.csmllibs.csmldirectory.DirectoryTree() |
---|
142 | tree.setTopDirectory(self.topdir) |
---|
143 | tree.readDirectory() |
---|
144 | subdirs = tree.getSubDirectories() |
---|
145 | ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() |
---|
146 | for dir in subdirs: |
---|
147 | files=tree.getCSMLSupportedFilesInSubDir(dir) |
---|
148 | for f in files: |
---|
149 | repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f) |
---|
150 | repfile.setFeatureType(self.ft) |
---|
151 | ffmap.addRepresentativeFile(repfile) |
---|
152 | return ffmap |
---|
153 | |
---|
154 | def oneonly(self): |
---|
155 | """ oneonly means one file represents feature spanning multiple directories |
---|
156 | assumes no file in toplevel directory, and then lots of subdirectories at next level containing files |
---|
157 | """ |
---|
158 | tree = csml.csmllibs.csmldirectory.DirectoryTree() |
---|
159 | tree.setTopDirectory(self.topdir) |
---|
160 | tree.readDirectory() |
---|
161 | subdirs = tree.getSubDirectories() |
---|
162 | allfiles= tree.getAllCSMLSupportedFiles() |
---|
163 | ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() |
---|
164 | |
---|
165 | #create representative file from first file |
---|
166 | repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(allfiles[0]) |
---|
167 | repfile.setFeatureType(self.ft) |
---|
168 | #create related files from all other files |
---|
169 | for f in allfiles[1:]: |
---|
170 | relfile = csml.csmllibs.csmlfeaturefilemap.relatedFile(f) |
---|
171 | relfile.setFeatureType(self.ft) |
---|
172 | repfile.addRelatedFile(relfile) |
---|
173 | ffmap.addRepresentativeFile(repfile) |
---|
174 | return ffmap |
---|
175 | |
---|
176 | |
---|
177 | |
---|
178 | |
---|
179 | def mixedft(self): |
---|
180 | ''' allows the use of a dictonary object to store feature file map info not well tested ''' |
---|
181 | tree = csml.csmllibs.csmldirectory.DirectoryTree() |
---|
182 | tree.setTopDirectory(self.topdir) |
---|
183 | tree.readDirectory() |
---|
184 | subdirs = tree.getSubDirectories() |
---|
185 | ffmap = csml.csmllibs.csmlfeaturefilemap.FeatureFileMap() |
---|
186 | for dir in subdirs: |
---|
187 | files=tree.getCSMLSupportedFilesInSubDir(dir) |
---|
188 | for f in files: |
---|
189 | repfile=csml.csmllibs.csmlfeaturefilemap.representativeFile(f) |
---|
190 | #look up feature type in dictionary |
---|
191 | ft=self.ftdictionary[f] |
---|
192 | repfile.setFeatureType(ft) |
---|
193 | ffmap.addRepresentativeFile(repfile) |
---|
194 | return ffmap |
---|