source: TI02-CSML/branches/csml-cdms2/csmlscan.py @ 3288

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI02-CSML/branches/csml-cdms2/csmlscan.py@3627
Revision 3288, 10.0 KB checked in by domlowe, 12 years ago (diff)

more tidying of scanner

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/env python
2
3#**************************************************************************************
4#cd2csml.py
5#read netcdf/pp files through cdunif/cdat and produce CSML documents
6#V0.56 Dominic Lowe, BADC 06th December 2005
7#V0.55 Dominic Lowe, BADC  08th November 2005
8#V0.5 Dominic Lowe, BADC  27th October 2005
9#
10#**************************************************************************************
11
12import sys, string
13import ConfigParser
14import getopt
15import os
16#CSML modules
17import csmllibs
18import parser
19import parser_extra
20
21
22
23usage = """Usage:
24    Coapec2CSML [options] <files>
25
26    Scan a list of files (netCDF/PP/GRIB) producing a CSML file for those files
27    (as a single CSML Dataset)
28    NOTE: these arguments are out of date.
29
30Arguments:
31
32    <files> is a list of file paths to scan. The files can be listed in any order,
33    and may be in multiple directories. Currently only full path names are supported.
34    These files will be marked up as a single CSML Dataset.
35       
36Options:
37   
38    c:d:i:a:f:xt:m:po:s:v:T:h", ["configfile=","directory=","id=","minaxes=", "csmlfeaturetype=", "exitafterconfig","timedimension=", "filemapping=",  "printscreen", "outputfile=", "spatialstorage=","valuestorage=","timestorage=","help"])
39
40    -c configfile: A config file as described on the CSML wiki here: http://proj.badc.rl.ac.uk/ndg/wiki/T02_CSML/Csmlscan#Examplecontentsofconfigfile
41   
42    -d directory: Root directory. Scanner will scan all files in this directory and below.
43   
44    -i id: Dataset ID. This will be written to the CSML file as the Dataset ID attribute. Otherwise a random ID is generated.
45   
46    -a minaxes: A dimension threshold below which variables are not considered feature. e.g. If this is 3 only 3-Dimensional and above variables will be considered for scanning. Useful for case where auxiliary variables may confuse the scanner.
47   
48    -f csmlfeaturetype: The csmlFeatureType of the data. Defaults to GridSeries.
49   
50    -x:   Print out the config options that have been set, but don't do the actual scan. Useful for trying out config options.
51                                               
52     -t timedimension: The default time dimension is 'time' as per the CF recomendations. If your files use a
53                different variable name you may supply it with this option.
54
55    -m filemapping: One of the file map patterns, see here for more info: http://proj.badc.rl.ac.uk/ndg/wiki/T02_CSML/Csmlscan#filessection:
56
57    -o outputfile:      The name of the output csml file, default is csmloutput.xml
58                                which can be found in the directory from which the program is run
59   
60    -p: If this flag is included, the CSML output is printed to std out.
61   
62    -s spatialstorage: How to store spatial domain info in the csml file: inline or fileextract
63   
64    -v valuestorage: How to store rangeset info in the csml file: inline or fileextract
65   
66    -T timestorage: How to store temporal info in the csml file: inline or fileextract
67   
68    -h help: Shows this help information.
69       
70       
71Examples:
72        csmlscan -o testout3.xml -t t /home/me/files/xaaqda@pxr99c1.nc /home/me/files/xaaqda@pxs09c1.nc
73               - processes 2 netcdf files supplied, which have a time dimension called 't'
74               - output csmlfile is called testout3.xml
75               
76        csmlscan -c myconfigfile.cfg
77                - reads settings from config file called myconfigfile.cfg
78               
79        TODO: more examples....
80       
81"""
82#this should go in ndg.utils
83def getConfigNoError(self, section, option):
84    # get the config option, returning None if not present
85    #value = self.get(section, option)
86    try:
87        value = self.get(section, option)
88    except ConfigParser.NoOptionError:
89        #config value is not set - option could not be found
90        value =None 
91    except ConfigParser.NoSectionError:
92        #config value is not set - section could not be found
93        value =None
94    return value
95   
96def main(optargs=None):
97    #assign extra method to configparser
98    ConfigParser.ConfigParser.getConfigNoError=getConfigNoError
99    config = ConfigParser.ConfigParser()
100    #Get command line arguments
101    #if len(sys.argv) ==1:
102        #print 'No arguments passed to csmlscan. Run "csmlscan --help" for more info.'
103        #raise SystemExit
104    if optargs:
105        #if called as main(args) from another python module use these args, else use
106        #sys.argv if called from command line.
107        sys.argv =optargs
108    try:
109            opts, args = getopt.getopt(sys.argv[1:], "c:d:i:a:f:xt:m:po:s:v:T:h", ["configfile=","directory=","id=","minaxes=", "csmlfeaturetype=", "exitafterconfig","timedimension=", "filemapping=",  "printscreen", "outputfile=", "spatialstorage=","valuestorage=","timestorage=","help"])
110    except getopt.error, msg:
111            print 'Invalid options, use "csmlscan --help" for help'
112            raise SystemExit
113    hasConfig=False
114    configTest=False
115    for o, v in opts:
116            if o in ("-c", "--configfile"):
117                configfile = v
118                hasConfig=True
119            elif o in ("-x", "--exitafterconfig"):
120                configTest=True
121           
122    #set default values for required configs
123    DATASETID=None
124    MINAXES = 2
125    PRINTSCREEN = 0
126    MAPPING=None
127    OUTPUTFILE='csmloutput.xml'
128    TIMEDIMENSION='time'
129    TIMESTORAGE='inline'
130    VALUESTORAGE='fileextract'
131    SPATIALSTORAGE='fileextract'
132    FEATURETYPE='GridSeries'
133    SELECTION=None
134    ROOTDIRECTORY=os.getcwd()
135   
136    #...
137    if hasConfig == True:
138        config.read(configfile)
139        conf=config.getConfigNoError('dataset', 'dsID')
140        if conf:
141            DATASETID = conf 
142        conf=config.getConfigNoError('features', 'type') 
143        if conf:
144            FEATURETYPE = conf
145        conf=config.getConfigNoError('features','minaxes')
146        if conf:
147            MINAXES=conf
148        conf=config.getConfigNoError('files', 'root')
149        if conf:
150            ROOTDIRECTORY = conf
151        conf=config.getConfigNoError('files', 'selection')
152        if conf:
153            SELECTION=conf
154        conf=config.getConfigNoError('files', 'mapping')
155        if conf:
156            MAPPING = conf
157        conf=config.getConfigNoError('files', 'output')
158        if conf:
159            OUTPUTFILE = conf
160        conf=config.getConfigNoError('files', 'printscreen')
161        if conf:
162            PRINTSCREEN = conf
163        conf=config.getConfigNoError('time','timedimension')
164        if conf:
165            TIMEDIMENSION=conf
166        conf=config.getConfigNoError('time','timestorage')
167        if conf:
168            TIMESTORAGE=conf
169        conf=config.getConfigNoError('values','valuestorage')
170        if conf:
171            VALUESTORAGE=conf
172        conf=config.getConfigNoError('spatialaxes','spatialstorage')
173        if conf:
174            SPATIALSTORAGE=conf
175   
176    else: #no config file, get command line args
177        for o, v in opts:
178            if o in ("-d", "--directory"):
179                ROOTDIRECTORY = v
180            elif o in ('-i','id'):
181                DATASETID=v
182            elif o in ('-a','minaxes'):
183                MINAXES=v
184            elif o in ("-f", "--csmlfeaturetype"):
185                FEATURETYPE = v
186            elif o in ("-t", "--timedimension"):
187                TIMEDIMENSION = v
188            elif o in ("-m", "--filemapping"):
189                MAPPING=v
190            elif o in ("-p", "--printscreen"):
191                PRINTSCREEN = 1
192            elif o in ("-o", "--outputfile"):
193                OUTPUTFILE = v
194            elif o in ("-T", "--timestorage"):
195                TIMESTORAGE = v
196            elif o in ("-v", "--valuestorage"):
197                VALUESTORAGE = v
198            elif o in ("-s", "--spatialstorage"):
199                SPATIALSTORAGE = v
200            elif o in ("-h", "--help"):
201                print usage
202                raise SystemExit
203        if args is not []:
204            SELECTION=args
205
206    # Handle defaults in case of no file mapping:
207    if MAPPING is None:
208        if FEATURETYPE == "Point":
209            MAPPING = 'onetoone' #default filemapping for PointFeature (?)
210        elif FEATURETYPE == "Profile":           
211            MAPPING = 'onetoone' #default filemapping for ProfileFeature (?)
212        elif FEATURETYPE == "Grid":
213            MAPPING = 'onetoone' #default filemapping for GridFeature
214        elif FEATURETYPE == "PointSeries":
215            MAPPING = 'onetomany'
216        elif FEATURETYPE == "ProfileSeries":
217            MAPPING = 'onetomany' #default filemapping for ProfileSeriesFeature (?)
218        elif FEATURETYPE == "GridSeries":
219            MAPPING = 'onetomany' #default filemapping for GridSeriesFeature     
220             
221        print '\n Starting scan...'
222        print '\n'
223        print 'The following config options have been set:'
224        print '\n dataset:dsID = %s'%DATASETID
225        print '\n features:type = %s'%FEATURETYPE
226        print '\n features:minaxes = %s'%MINAXES
227        print '\n files:root = %s'%ROOTDIRECTORY
228        print '\n files.selection = %s'%SELECTION
229        print '\n files:mapping = %s'%MAPPING
230        print '\n files:output = %s'%OUTPUTFILE
231        print '\n files:printscreen = %s'%PRINTSCREEN
232        print '\n time:timedimension = %s'%TIMEDIMENSION
233        print '\n time:timestorage = %s'%TIMESTORAGE
234        print '\n spatialaxes:spatialstorage = %s'%SPATIALSTORAGE
235        print '\n values:valuestorage = %s'%VALUESTORAGE
236        print '\n For more information on config options run the command "csmlscan --help"'
237       
238       
239        if configTest==True:
240            #don't actually start the scan
241            print '\n Abandoning scan, remove "-x" flag to scan for real'
242            raise SystemExit
243           
244           
245       
246    #build CSML document
247    csmldataset=csmllibs.csmlbuilder.csmlBuilder(DATASETID,ROOTDIRECTORY,SELECTION,FEATURETYPE,MAPPING,TIMEDIMENSION, OUTPUTFILE, PRINTSCREEN,TIMESTORAGE,SPATIALSTORAGE,VALUESTORAGE, MINAXES)
248    csmldataset.build()
249
250    print ''
251    print '********************************************************************'
252    print 'CSML file is at: ' + OUTPUTFILE
253    print '********************************************************************'               
254    print ''
255
256
257if __name__=='__main__':   
258    main()
259
260
261
262
263
264
Note: See TracBrowser for help on using the repository browser.