source: TI02-CSML/trunk/csml/csmlscan.py @ 3856

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI02-CSML/trunk/csml/csmlscan.py@3856
Revision 3856, 10.3 KB checked in by domlowe, 12 years ago (diff)

Adding security options back in to scanner

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/env python
2
3#**************************************************************************************
4#cd2csml.py
5#read netcdf/pp files through cdunif/cdat and produce CSML documents
6#V0.56 Dominic Lowe, BADC 06th December 2005
7#V0.55 Dominic Lowe, BADC  08th November 2005
8#V0.5 Dominic Lowe, BADC  27th October 2005
9#
10#**************************************************************************************
11
12import sys, string
13import ConfigParser
14import getopt
15import os
16#CSML modules
17import csmllibs
18import parser
19import parser_extra
20
21
22
23usage = """Usage:
24    Coapec2CSML [options] <files>
25
26    Scan a list of files (netCDF/PP/GRIB) producing a CSML file for those files
27    (as a single CSML Dataset)
28    NOTE: these arguments are out of date.
29
30Arguments:
31
32    <files> is a list of file paths to scan. The files can be listed in any order,
33    and may be in multiple directories. Currently only full path names are supported.
34    These files will be marked up as a single CSML Dataset.
35       
36Options:
37   
38    c:d:i:a:f:xt:m:po:s:v:T:h", ["configfile=","directory=","id=","minaxes=", "csmlfeaturetype=", "exitafterconfig","timedimension=", "filemapping=",  "printscreen", "outputfile=", "spatialstorage=","valuestorage=","timestorage=","help"])
39
40    -c configfile: A config file as described on the CSML wiki here: http://proj.badc.rl.ac.uk/ndg/wiki/T02_CSML/Csmlscan#Examplecontentsofconfigfile
41   
42    -d directory: Root directory. Scanner will scan all files in this directory and below.
43   
44    -i id: Dataset ID. This will be written to the CSML file as the Dataset ID attribute. Otherwise a random ID is generated.
45   
46    -a minaxes: A dimension threshold below which variables are not considered feature. e.g. If this is 3 only 3-Dimensional and above variables will be considered for scanning. Useful for case where auxiliary variables may confuse the scanner.
47   
48    -f csmlfeaturetype: The csmlFeatureType of the data. Defaults to GridSeries.
49   
50    -x:   Print out the config options that have been set, but don't do the actual scan. Useful for trying out config options.
51                                               
52     -t timedimension: The default time dimension is 'time' as per the CF recomendations. If your files use a
53                different variable name you may supply it with this option.
54
55    -m filemapping: One of the file map patterns, see here for more info: http://proj.badc.rl.ac.uk/ndg/wiki/T02_CSML/Csmlscan#filessection:
56
57    -o outputfile:      The name of the output csml file, default is csmloutput.xml
58                                which can be found in the directory from which the program is run
59   
60    -p: If this flag is included, the CSML output is printed to std out.
61   
62    -s spatialstorage: How to store spatial domain info in the csml file: inline or fileextract
63   
64    -v valuestorage: How to store rangeset info in the csml file: inline or fileextract
65   
66    -T timestorage: How to store temporal info in the csml file: inline or fileextract
67   
68    -h help: Shows this help information.
69       
70       
71Examples:
72        csmlscan -o testout3.xml -t t /home/me/files/xaaqda@pxr99c1.nc /home/me/files/xaaqda@pxs09c1.nc
73               - processes 2 netcdf files supplied, which have a time dimension called 't'
74               - output csmlfile is called testout3.xml
75               
76        csmlscan -c myconfigfile.cfg
77                - reads settings from config file called myconfigfile.cfg
78               
79        TODO: more examples....
80       
81"""
82#this should go in ndg.utils
83def getConfigNoError(self, section, option):
84    # get the config option, returning None if not present
85    #value = self.get(section, option)
86    try:
87        value = self.get(section, option)
88    except ConfigParser.NoOptionError:
89        #config value is not set - option could not be found
90        value =None 
91    except ConfigParser.NoSectionError:
92        #config value is not set - section could not be found
93        value =None
94    return value
95   
96def main(optargs=None):
97    #assign extra method to configparser
98    ConfigParser.ConfigParser.getConfigNoError=getConfigNoError
99    config = ConfigParser.ConfigParser()
100    #Get command line arguments
101    #if len(sys.argv) ==1:
102        #print 'No arguments passed to csmlscan. Run "csmlscan --help" for more info.'
103        #raise SystemExit
104    if optargs:
105        #if called as main(args) from another python module use these args, else use
106        #sys.argv if called from command line.
107        sys.argv =optargs
108    try:
109            opts, args = getopt.getopt(sys.argv[1:], "c:d:i:a:f:xt:m:po:s:v:T:h", ["configfile=","directory=","id=","minaxes=", "csmlfeaturetype=", "exitafterconfig","timedimension=", "filemapping=",  "printscreen", "outputfile=", "spatialstorage=","valuestorage=","timestorage=","help"])
110    except getopt.error, msg:
111            print 'Invalid options, use "csmlscan --help" for help'
112            raise SystemExit
113    hasConfig=False
114    configTest=False
115    for o, v in opts:
116            if o in ("-c", "--configfile"):
117                configfile = v
118                hasConfig=True
119            elif o in ("-x", "--exitafterconfig"):
120                configTest=True
121           
122    #set default values for required configs
123    DATASETID=None
124    MINAXES = 2
125    PRINTSCREEN = 0
126    MAPPING=None
127    OUTPUTFILE='csmloutput.xml'
128    TIMEDIMENSION='time'
129    TIMESTORAGE='inline'
130    VALUESTORAGE='fileextract'
131    SPATIALSTORAGE='fileextract'
132    FEATURETYPE='GridSeries'
133    SELECTION=None
134    ROOTDIRECTORY=os.getcwd()
135    SEC_ALLOW=None
136    SEC_DENY=None
137   
138    #...
139    if hasConfig == True:
140        config.read(configfile)
141        conf=config.getConfigNoError('dataset', 'dsID')
142        if conf:
143            DATASETID = conf 
144        conf=config.getConfigNoError('features', 'type') 
145        if conf:
146            FEATURETYPE = conf
147        conf=config.getConfigNoError('features','minaxes')
148        if conf:
149            MINAXES=conf
150        conf=config.getConfigNoError('files', 'root')
151        if conf:
152            ROOTDIRECTORY = conf
153        conf=config.getConfigNoError('files', 'selection')
154        if conf:
155            SELECTION=conf
156        conf=config.getConfigNoError('files', 'mapping')
157        if conf:
158            MAPPING = conf
159        conf=config.getConfigNoError('files', 'output')
160        if conf:
161            OUTPUTFILE = conf
162        conf=config.getConfigNoError('files', 'printscreen')
163        if conf:
164            PRINTSCREEN = conf
165        conf=config.getConfigNoError('time','timedimension')
166        if conf:
167            TIMEDIMENSION=conf
168        conf=config.getConfigNoError('time','timestorage')
169        if conf:
170            TIMESTORAGE=conf
171        conf=config.getConfigNoError('values','valuestorage')
172        if conf:
173            VALUESTORAGE=conf
174        conf=config.getConfigNoError('spatialaxes','spatialstorage')
175        if conf:
176            SPATIALSTORAGE=conf
177        conf=config.getConfigNoError('security', 'allow')
178        if conf:
179            SEC_ALLOW=conf
180        conf=config.getConfigNoError('security', 'deny')
181        if conf:
182            SEC_DENY=conf
183   
184    else: #no config file, get command line args
185        for o, v in opts:
186            if o in ("-d", "--directory"):
187                ROOTDIRECTORY = v
188            elif o in ('-i','id'):
189                DATASETID=v
190            elif o in ('-a','minaxes'):
191                MINAXES=v
192            elif o in ("-f", "--csmlfeaturetype"):
193                FEATURETYPE = v
194            elif o in ("-t", "--timedimension"):
195                TIMEDIMENSION = v
196            elif o in ("-m", "--filemapping"):
197                MAPPING=v
198            elif o in ("-p", "--printscreen"):
199                PRINTSCREEN = 1
200            elif o in ("-o", "--outputfile"):
201                OUTPUTFILE = v
202            elif o in ("-T", "--timestorage"):
203                TIMESTORAGE = v
204            elif o in ("-v", "--valuestorage"):
205                VALUESTORAGE = v
206            elif o in ("-s", "--spatialstorage"):
207                SPATIALSTORAGE = v
208            elif o in ("-h", "--help"):
209                print usage
210                raise SystemExit
211        if args is not []:
212            SELECTION=args
213
214    # Handle defaults in case of no file mapping:
215    if MAPPING is None:
216        if FEATURETYPE == "Point":
217            MAPPING = 'onetoone' #default filemapping for PointFeature (?)
218        elif FEATURETYPE == "Profile":           
219            MAPPING = 'onetoone' #default filemapping for ProfileFeature (?)
220        elif FEATURETYPE == "Grid":
221            MAPPING = 'onetoone' #default filemapping for GridFeature
222        elif FEATURETYPE == "PointSeries":
223            MAPPING = 'onetomany'
224        elif FEATURETYPE == "ProfileSeries":
225            MAPPING = 'onetomany' #default filemapping for ProfileSeriesFeature (?)
226        elif FEATURETYPE == "GridSeries":
227            MAPPING = 'onetomany' #default filemapping for GridSeriesFeature     
228             
229        print '\n Starting scan...'
230        print '\n'
231        print 'The following config options have been set:'
232        print '\n dataset:dsID = %s'%DATASETID
233        print '\n features:type = %s'%FEATURETYPE
234        print '\n features:minaxes = %s'%MINAXES
235        print '\n files:root = %s'%ROOTDIRECTORY
236        print '\n files.selection = %s'%SELECTION
237        print '\n files:mapping = %s'%MAPPING
238        print '\n files:output = %s'%OUTPUTFILE
239        print '\n files:printscreen = %s'%PRINTSCREEN
240        print '\n time:timedimension = %s'%TIMEDIMENSION
241        print '\n time:timestorage = %s'%TIMESTORAGE
242        print '\n spatialaxes:spatialstorage = %s'%SPATIALSTORAGE
243        print '\n values:valuestorage = %s'%VALUESTORAGE
244        print '\n security:allow = %s'%SEC_ALLOW
245        print '\n security:deny = %s'%SEC_DENY
246        print '\n For more information on config options run the command "csmlscan --help"'
247       
248       
249        if configTest==True:
250            #don't actually start the scan
251            print '\n Abandoning scan, remove "-x" flag to scan for real'
252            raise SystemExit
253           
254           
255       
256    #build CSML document
257    csmldataset=csmllibs.csmlbuilder.csmlBuilder(DATASETID,ROOTDIRECTORY,SELECTION,FEATURETYPE,MAPPING,TIMEDIMENSION, OUTPUTFILE, PRINTSCREEN,TIMESTORAGE,SPATIALSTORAGE,VALUESTORAGE, MINAXES, SEC_ALLOW, SEC_DENY)
258    csmldataset.build()
259
260    print ''
261    print '********************************************************************'
262    print 'CSML file is at: ' + OUTPUTFILE
263    print '********************************************************************'               
264    print ''
265
266
267if __name__=='__main__':   
268    main()
269
270
271
272
273
274
Note: See TracBrowser for help on using the repository browser.