source: TI03-DataExtractor/branches/old_stuff/dx_badc/shelve_datasets.py @ 793

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/branches/old_stuff/dx_badc/shelve_datasets.py@793
Revision 793, 6.5 KB checked in by astephen, 13 years ago (diff)

Put all the old code in the old_stuff branch.

  • Property svn:executable set to *
Line 
1#!/usr/local/cdat/bin/python
2
3"""
4shelve_datasets.py
5==================
6
7Additional shelving command-line utility for the extractor package.
8
9This module reads a CDML (XML) file and extracts the required information.
10
11Version history
12===============
13
14Version 1.0:  - 01/03/2004
15              - Ag Stephens, British Atmospheric Data Centre.
16              - First version.
17
18"""
19
20# Import required modules
21
22import shelve
23import cdms
24import os
25
26
27# Define global variables
28e40xml = "/badc/ecmwf-e40/.cdml"
29datasets={"ERA-40":"era4t", "Ag's data":None}  # Name and security ID
30subsets={"ERA-40":(("Analyses on Surface/Single Levels (1.0 deg)", "/badc/ecmwf-e40/.cdml/e40-1.0-as-1958-2001.xml", "e40-1.0-as"),
31                              ("Analyses on Pressure Levels (1.0 deg)", "/badc/ecmwf-e40/.cdml/e40-1.0-ap-1958-2001.xml", "e40-1.0-ap"), 
32                              ("Analyses on Model Levels (1.0 deg)", "/badc/ecmwf-e40/.cdml/e40-1.0-am-1958-2001.xml", "e40-1.0-am"), 
33                              ("Analyses on Model Levels (Spectral T159)", "/badc/ecmwf-e40/.cdml/e40-t159-am-1958-2001.xml", "e40-t159-am"),
34                              ("Analyses on Pressure Levels (Spectral T159)", "/badc/ecmwf-e40/.cdml/e40-t159-ap-1958-2001.xml", "e40-t159-ap"),
35                              ("Analyses on Potential Temperature Levels (Spectral T159)", "/badc/ecmwf-e40/.cdml/e40-t159-at-1958-2001.xml", "e40-t159-at")),
36                    "Ag's data":(("Surface Level Analyses (1.0 deg)", "/var/www/cgi-bin/extractor/data/e40-1.0-as-1958-2001.xml", "ag-1.0-as"),)}
37
38
39outdir="/var/www/cgi-bin/extractor/datasets"
40
41
42def exitNicely(message):
43    pass
44
45class SubsetDict:
46
47    def __init__(self, subset):
48        self.dict={}
49        self.dict["id"]=subset[2]
50        self.dict["long_name"]=subset[0]
51        self.dict["source_xml"]=subset[1]
52
53        self.file=cdms.open(self.dict["source_xml"])
54        self.populateFile()
55
56    def populateFile(self):
57        variables=self.file.getVariables()
58        self.dict["variables"]=[]
59
60        for var in variables:
61            (long_name, short_name, units)=(var.title, var.id, var.units)
62            self.dict["variables"].append((short_name, long_name, units))
63
64        var1=variables[0]         
65        time_axis=var1.getTime()
66       
67        # Get the start and end date and time
68        import cdtime
69        cdtime_keys=("year", "month", "day", "hour", "minute", "second")
70        start_time_components=cdtime.r2c(cdtime.reltime(time_axis[0], time_axis.units))
71        end_time_components=cdtime.r2c(cdtime.reltime(time_axis[-1], time_axis.units))
72        start_time=[]
73        end_time=[]
74        for key in cdtime_keys:
75            start_time.append(getattr(start_time_components, key))
76            end_time.append(getattr(end_time_components, key))
77
78        self.dict["start_time"]=tuple(start_time)
79        self.dict["end_time"]=tuple(end_time)
80        self.dict["time_interval"]=(abs(time_axis[1]-time_axis[0]), time_axis.units)
81
82        lat=var1.getLatitude()[:]
83        (south, north)=(lat[0], lat[-1])
84
85        lon=var1.getLongitude()[:]
86        (west, east)=(lon[0], lon[-1])
87
88        resolution=(abs(lat[1]-lat[0]), abs(lon[1]-lon[0]))
89
90        self.dict["horiz_domain"]=(north, west, south, east)
91        self.dict["resolution"]=resolution
92
93        try:
94            level=var1.getLevel()
95            self.dict["vertical_units"]=level.units
96            self.dict["vertical_domain"]=tuple(level[:])
97            #print "<H1>%s ------------- %s" % (self.dict["vertical_domain"], self.dict["vertical_units"])
98        except:
99            self.dict["vertical_domain"]=("Single level",)
100            self.dict["vertical_units"]=None       
101 
102        #print "<H1>%s ------------- %s" % (self.dict["vertical_domain"], self.dict["vertical_units"])
103         
104
105if __name__=="__main__":
106  quick=0
107
108  if quick==1:
109    """
110    dict={"datasets":{"Ag's data":{}, "ERA-40":{}, "ERA-15":{}, "Unified Model (UKMO)":{}}}
111    dict["datasets"]["ERA-40"]["id"]="ERA-40"
112    dict["datasets"]["ERA-40"]["subsets"]={"Surface Level Analyses (1.0 deg)":{},
113         "Pressure Level Analyses (1.0 deg)":{},
114         "Model Level Analyses (1.0 deg)":{}}
115 
116    dict["datasets"]["Ag's data"]["id"]="Ag's data"
117    dict["datasets"]["Ag's data"]["subsets"]={"Surface Level Analyses (1.0 deg)":{},
118                                "Pressure Level Analyses (1.0 deg)":{},
119                                "Model Level Analyses (1.0 deg)":{}}
120
121    subset={}
122    subset["id"]="Surface Level Analyses (1.0 deg)"
123    subset["source_xml"]="/badc/ecmwf-e40/.cdml/e40-1.0-as-1958-2001.xml"
124    subset["variables"]=(("tcc", "Total Cloud Cover", "(0-1)"),
125                         ("10u", "10-metre U-wind", "m s**-1"))
126    subset["start_time"]=(1958, 1, 1, 0, 0, 0)
127    subset["end_time"]=(2001, 12, 31, 18, 0, 0)
128    subset["time_interval"]=(6, "hr")
129    subset["horiz_domain"]=(90.0, 0.0, -90.0, 359.0)
130    subset["resolution"]=(1.0, 1.0)
131    subset["vertical_domain"]=("Surface/single levels")
132    subset["vertical_units"]=None
133
134    dict["datasets"]["ERA-40"]["Surface Level Analyses (1.0 deg)"]=subset
135
136    subset={}
137    subset["id"]="Surface Level Analyses (1.0 deg)"
138    subset["source_xml"]="/var/www/cgi-bin/extractor/data/e40-1.0-as-1958-2001.xml"
139    subset["variables"]=(("tcc", "Total Cloud Cover", "(0-1)"),
140                         ("10u", "10-metre U-wind", "m s**-1"))
141    subset["start_time"]=(1999, 1, 1, 0, 0, 0)
142    subset["end_time"]=(1999, 1, 1, 18, 0, 0)
143    subset["time_interval"]=(6, "hr")
144    subset["horiz_domain"]=(90.0, 0.0, -90.0, 359.0)
145    subset["resolution"]=(1.0, 1.0)
146    subset["vertical_domain"]=("Surface/single levels")
147    subset["vertical_units"]=None   
148 
149    dict["datasets"]["Ag's data"]["Surface Level Analyses (1.0 deg)"]=subset
150   
151    shelve_obj=shelve.open(os.path.join(outdir, "datasetdb"))
152
153    for key in dict.keys():
154        shelve_obj[key]=dict[key]
155
156    shelve_obj.close()
157    """
158
159  else:
160    topdict={}
161    topdict["datasets"]={}
162    for d in datasets.keys():
163        topdict["datasets"][d]={}
164        topdict["datasets"][d]["id"]=d
165        topdict["datasets"][d]["protect_id"]=datasets[d]
166        topdict["datasets"][d]["subsets"]={}
167
168        for s in subsets[d]:
169            (long_name, xml, code)=s       
170            print "Ingesting %s -> %s" % (d, long_name)
171            subsetter=SubsetDict(s)
172            topdict["datasets"][d]["subsets"][long_name]=subsetter.dict
173
174    shelve_obj=shelve.open(os.path.join(outdir, "datasetdb"))
175    shelve_obj["datasets"]=topdict["datasets"]
176    shelve_obj.close()
177   
178  print "'datasetdb' shelve object has been written successfully."
Note: See TracBrowser for help on using the repository browser.