Changeset 1244 for TI03-DataExtractor/trunk
- Timestamp:
- 27/06/06 11:04:47 (15 years ago)
- Location:
- TI03-DataExtractor/trunk
- Files:
-
- 303 added
- 14 edited
Legend:
- Unmodified
- Added
- Removed
-
TI03-DataExtractor/trunk/cgi/dxui
r1225 r1244 15 15 16 16 # Import standard library modules 17 import cgi, random, time, re 17 import cgi, random, time, re, os, sys 18 18 19 19 # Import SOAP library … … 22 22 zsiv=float("%s.%s" % zsiv[:2]) 23 23 24 # Get configuration file for this dx client 25 configFile="<PLACE_CONFIG_PATH_HERE>" 26 configPath, configFilename=os.path.split(configFile) 27 sys.path.insert(0, configPath) 28 exec ("from %s import *" % configFilename[:-3]) 29 24 30 # Import package modules 25 31 from pydxc import * … … 234 240 235 241 elif SECURITY_MODEL=="ndg": 236 from NDGSecurityViaCGI import *242 from pydxc.NDGSecurityViaCGI import * 237 243 secChecker=NDGSecurityViaCGI(cookie=self.secureToken, urlArgs=self.fieldStorage) 238 239 244 self.loginlist=secChecker.getTrustedHostList() 245 246 #(self.secureToken, self.username, self.userRoles)= 247 secCheck=secChecker.validate() 248 if type(secCheck)==type(""): 249 self.loginMessage=secCheck 250 else: 251 self.loginStatus="in" 252 self.loginMessage="" 253 (self.secureToken, self.username, self.userRoles)=secCheck 254 255 #o=open("/tmp/t", "w"); o.write("%s" % repr(secCheck)); o.close() 240 256 241 257 # Delete field storage so not visible anywhere 242 del self.fieldStorage258 #del self.fieldStorage 243 259 244 260 … … 248 264 Web Service and calls it with the relevant arguments. 249 265 """ 266 #print "Content-Type: text/html\n\n" 250 267 # Just print the login page if not logged in and login required 251 268 if RESTRICTED_DATA==1 and self.loginStatus=="out": 252 269 self.displayer._displayHTTPHeader() 253 270 self.displayer._displayHTMLHeader() 254 self.displayer._displayIntroduction() 255 self.displayer._displayLoginBar(self.username, self.loginStatus, self.loginMessage) 271 #print "FIELD STORAGE:", self.fieldStorage 272 self.displayer._displayIntroduction() 273 #print self.loginlist 274 if SECURITY_MODEL=="basic": 275 self.displayer._displayLoginBar(self.username, self.loginStatus, self.loginMessage) 276 elif SECURITY_MODEL=="ndg": 277 argString="?" 278 for key in self.fieldStorage.keys(): 279 if key.find("datasetURI_")>-1: 280 if argString!="?": 281 gapper="&" 282 else: 283 gapper="" 284 argString=argString+("%s%s=%s" % (gapper, key, self.fieldStorage.getvalue(key))) 285 if argString=="?": argString="" 286 self.displayer._displayNDGLoginBar(self.loginlist, argString) 287 # Delete field storage so not visible anywhere 288 del self.fieldStorage 256 289 return 257 290 258 291 # Set up SOAP bindings 259 292 if CALL_METHOD.upper()=="WS": … … 424 457 optcat=optionCategories[0].split("_")[0] 425 458 459 if optcat==None and (RESTRICTED_DATA==1 and summaryString.find("dataset")<0): 460 self.displayer._displayHTTPHeader() 461 self.displayer._displayHTMLHeader() 462 print "<P><B>You do not have the credential to view any datasets - Sorry!</B><P>" 463 426 464 # Can display simple form for these categories 427 if optcat in ("datasetGroup", "dataset", "variable"):465 elif optcat in ("datasetGroup", "dataset", "variable"): 428 466 self.displayer._displayHTTPHeader() 429 467 self.displayer._displayHTMLHeader() -
TI03-DataExtractor/trunk/dxc/scripts/exampleCLClient1.py
r1225 r1244 1 1 #!/usr/bin/env python 2 import sys 3 import re 2 import sys, re, os 3 4 # Get configuration file as first argument 5 args=sys.argv[1:] 6 if len(args)!=1: 7 print "ERROR: please provide the location of the configuration file as the only argument to this script." 8 sys.exit() 9 10 configFile=args[0] 11 configPath, configFilename=os.path.split(configFile) 12 sys.path.insert(0, configPath) 13 exec ("from %s import *" % configFilename[:-3]) 14 15 4 16 #import readline 5 17 from ZSI.client import Binding 6 18 from pydxc.common import * 7 from pydxc.clientConfig import SOAP_SERVER_URL 19 sys.path.append("../configs") 20 sys.path.append("dxc/configs") 21 sys.path.append("configs") 22 #from clientConfig import SOAP_SERVER_URL 8 23 9 24 urlPattern=re.compile(r"(\w+)://([\w.]+):(\d+)/(.*)$") -
TI03-DataExtractor/trunk/dxs/bin/DXWSInterface.py
r1225 r1244 35 35 36 36 # Get configuration file as first argument 37 #args=sys.argv[1:]38 #if len(args)!=1:39 #print "ERROR: please provide the location of the configuration file as the only argument to this script."40 #sys.exit()41 42 #configFile=args[0]43 #configPath, configFilename=os.path.split(configFile)44 #sys.path.insert(0, configPath)45 #exec ("from %s import *" % configFilename[:-3])37 args=sys.argv[1:] 38 if len(args)!=1: 39 print "ERROR: please provide the location of the configuration file as the only argument to this script." 40 sys.exit() 41 42 configFile=args[0] 43 configPath, configFilename=os.path.split(configFile) 44 sys.path.insert(0, configPath) 45 exec ("from %s import *" % configFilename[:-3]) 46 46 47 47 from ZSI import dispatch … … 54 54 from pydxs.common import * 55 55 from pydxs.DXController import * 56 from pydxs.serverConfig import *56 #from pydxs.serverConfig import * 57 57 from pydxs.DXRMLHandler import * 58 58 from pydxs.DXErrors import * -
TI03-DataExtractor/trunk/dxs/datasets/inputDatasets.xml
r1225 r1244 424 424 425 425 <dxGroupLongName> 426 CSML test dataset group426 The CSML test dataset group 427 427 </dxGroupLongName> 428 428 -
TI03-DataExtractor/trunk/dxs/testdata/testdata1.xml
r794 r1244 3 3 <dataset 4 4 Conventions ="CF-1.0" 5 cdms_filemap ="[[[bounds_latitude,bounds_longitude],[[-,-,-,-,var1.nc]]],[[pqn ],[[0,2,-,-,var1.nc]]]]"5 cdms_filemap ="[[[bounds_latitude,bounds_longitude],[[-,-,-,-,var1.nc]]],[[pqn,twotimespqn_dxvv],[[0,2,-,-,var1.nc]]]]" 6 6 directory ="" 7 7 calendar ="proleptic_gregorian" … … 98 98 </domain> 99 99 </variable> 100 <variable 101 datatype ="Double" 102 long_name ="A test virtual variable that will not output yet" 103 id ="twotimespqn_dxvv" 104 > 105 <attr datatype="String" name="name">twotimespqn_dxvv</attr> 106 <domain 107 > 108 <domElem start="0" length="2" name="time"/> 109 <domElem start="0" length="37" name="latitude"/> 110 <domElem start="0" length="72" name="longitude"/> 111 </domain> 112 </variable> 100 113 </dataset> -
TI03-DataExtractor/trunk/pydxc/DisplayManager.py
r1184 r1244 81 81 """ 82 82 if RESTRICTED_DATA: 83 statusMap={"out":"not logged in", "in":"logged in as %s" % username 83 statusMap={"out":"not logged in", "in":"logged in as %s" % username} 84 84 print "<P><B>Login status: </B>%s " % statusMap[loginStatus] 85 85 if len(loginMessage)>0: … … 95 95 print """<FORM NAME="loginForm" method="POST" action="%s"> 96 96 <INPUT TYPE="submit" NAME="logout" VALUE="Logout"></FORM>""" % CGI_SCRIPT_URL_PATH 97 98 99 def _displayNDGLoginBar(self, loginHosts, uriArgString):#, username, loginStatus, loginMessage=""): 100 """ 101 If security applies then display the login information. 102 """ 103 if RESTRICTED_DATA: 104 print """<P>Login:<P> 105 <form action="https://glue.badc.rl.ac.uk/cgi-bin/security.py" method="POST"> 106 <table><tr> 107 <td><select name="requestURI"> 108 <option value="">Select your home site...""" 109 110 for i,j in loginHosts: 111 print """<option value="%s"/>%s""" % (j,i) 112 113 print """</select></td> 114 <td><input type="submit" value="Login"></td> 115 </tr></table> 116 117 <input type="hidden" name="returnURI" value="%s%s"> 118 </form> 119 """ % (CGI_SCRIPT_URL_PATH, uriArgString) 120 # statusMap={"out":"not logged in", "in":"logged in as %s" % username} 121 # print "<P><B>Login status: </B>%s " % statusMap[loginStatus] 122 # if len(loginMessage)>0: 123 # print '<FONT COLOR="red"> [ %s ]</FONT>' % loginMessage 124 # if loginStatus=="out": 125 # # Display a login form 126 # print """<FORM NAME="loginForm" method="POST" action="%s"> 127 # Username: <INPUT TYPE="text" SIZE="20" NAME="yousirnaim" VALUE=""> 128 # Password: <INPUT TYPE="password" SIZE="10" NAME="parcewerd" VALUE=""> 129 # <INPUT TYPE="submit" NAME="login" VALUE="Login"></FORM>""" % CGI_SCRIPT_URL_PATH 130 # elif loginStatus=="in": 131 # # Display a logout form 132 # print """<FORM NAME="loginForm" method="POST" action="%s"> 133 # <INPUT TYPE="submit" NAME="logout" VALUE="Logout"></FORM>""" % CGI_SCRIPT_URL_PATH 97 134 98 135 … … 226 263 227 264 itemTitle=itemMap[optionCategories[0].split("_")[0]] 228 print '<P><B>PLEASE SELECT: %s</B> ( <A NAME="selectAll" onclick="selectAllCheckBoxes()">select all</A> | <A NAME="deSelectAll" onclick="deSelectAllCheckBoxes()">deselect all</A> )<P>' % itemTitle265 print '<P><B>PLEASE SELECT: %s</B> (<A NAME="selectAll" onclick="selectAllCheckBoxes()"> select all </A> / <A NAME="deSelectAll" onclick="deSelectAllCheckBoxes()"> deselect all </A> )<P>' % itemTitle 229 266 230 267 # Insert useful select all javascript function for checkboxes … … 308 345 pass 309 346 347 if itemName[-5:]=="_dxvv": 348 itemValue=itemValue+" (Virtual Variable)" 349 310 350 (dsgKey, dsKey)=re.match("variable_(\d+)\.(\d+)\.", selectionID).groups() 311 351 datasetGroup=summaryDict["datasetGroups"][dsgKey] … … 338 378 formatDict[item]=options[counter] 339 379 counter=counter+1 340 380 381 """sameBucket=getIdenticalDomains(axisBucket) 382 if len(sameBucket)>0: 383 print "<P>Found the same: " 384 for sameVars in sameBucket: 385 print "<BR>", (" ".join(sameVars)) 386 print "<P>" """ 387 341 388 (optionCategories, options, optionStrings)=axisBucket 342 389 #print "<P>", formatDict, "<P>", optionCategories … … 1213 1260 1214 1261 print " Note that you should choose NetCDF format if you wish to visualise data." 1215 print '<P><A HREF="%s?action=saveRequest&sessionID=%s">Get Request XML only.</A>' % (CGI_SCRIPT_URL_PATH, sessionID)1262 #print '<P><A HREF="%s?action=saveRequest&sessionID=%s">Get Request XML only.</A>' % (CGI_SCRIPT_URL_PATH, sessionID) 1216 1263 print '<P><INPUT TYPE="submit" NAME="proceed" VALUE="Proceed"><P>' 1217 1264 print "</FORM><P>" -
TI03-DataExtractor/trunk/pydxc/NDGSecurityViaCGI.py
r1184 r1244 12 12 13 13 # Import python standard library modules 14 import sys, os, time, Cookie, string 14 import sys, os, time, Cookie, string, re 15 15 16 16 from NDG.SecurityClient import * … … 19 19 from clientConfig import COOKIE_NAME, TOKEN_VALID_LIFETIME, TOKEN_DOMAIN 20 20 21 21 ct="Content-type: text/html\n\n" 22 22 class NDGSecurityViaCGI: 23 23 """ … … 29 29 Initialises the instance defining instance variables 30 30 """ 31 sys.path.append("/disks/glue1/astephens") 32 import stuff 33 self.username=None 34 self.roles=None 35 self.conf=stuff 31 36 self.cookie=cookie 32 # convert urlArgs into stuff we need 37 self.ndgcookie=None 38 self.urlArgs=urlArgs 33 39 self.ndgSec=[] 34 if urlArgs!=None: 35 keys=urlArgs.keys() 40 41 def getUsernameAndRolesFromNDGLogin(self): 42 found=0 43 storedCookies=os.environ.get("HTTP_COOKIE") 44 if storedCookies: 45 if storedCookies.find("NDG-ID1")>-1 and storedCookies.find("NDG-ID2")>-1: 46 #print ct #,"Coooooooooook", storedCookies 47 ndgID1=self._readCookie("NDG-ID1") 48 ndgID2=self._readCookie("NDG-ID2") 49 self.ndgSec.append(ndgID1) 50 self.ndgSec.append(ndgID2) 51 found=1 52 elif self.urlArgs!=None: 53 keys=self.urlArgs.keys() 36 54 if "NDG-ID1" in keys and "NDG-ID2" in keys: 37 self.ndgSec[0]=urlArgs.getvalue("NDG-ID1") 38 self.ndgSec[1]=urlArgs.getvalue("NDG-ID2") 39 55 self.ndgSec.append(self.urlArgs["NDG-ID1"].value) 56 self.ndgSec.append(self.urlArgs["NDG-ID2"].value) 57 found=1 58 59 if found==1: 60 self.smClient = SessionClient(smWSDL=self.conf.sessionMgrURL, 61 smPubKeyFilePath=self.conf.localSessionManagerPublicKey, 62 clntPubKeyFilePath=self.conf.thisCGIpublicKey, 63 clntPriKeyFilePath=self.conf.thisCGIprivateKey) 64 resp=self.smClient.reqAuthorisation(sessID=self.ndgSec[0], encrSessMgrWSDLuri=self.ndgSec[1], 65 reqRole="coapec", aaWSDL=self.conf.aaWSDL, 66 mapFromTrustedHosts=True, clntPriKeyPwd=None) 67 68 try: 69 ac=resp["attCert"] 70 self.roles=ac.getRoles() 71 holder=ac.getHolder() 72 realHolder=re.match("/CN=(\w+)/O", holder).group(1) 73 self.username=realHolder 74 except: 75 if type(resp)!=type("hi"): resp=repr(resp) 76 if resp.find("is before Attribute Certificate's not before time")>-1: return "Attribute Certificate 'not before time' error detected." 77 78 79 80 81 def getTrustedHostList(self): 82 self.aa=AttAuthorityClient(aaWSDL=self.conf.aaWSDL) 83 #print ct, dir(self.aa), self.conf.aaWSDL 84 thd=self.aa.getTrustedHostInfo(role="university") 85 self.loginHosts=[] 86 for key in thd.keys(): 87 self.loginHosts.append((key, thd[key]["loginURI"])) 88 return self.loginHosts 40 89 41 90 def validate(self): 42 91 """ 43 Returns either a None type meaning that the user is not44 valid, or a tuple of (secureToken, knownRoles).92 Returns either a string with a message meaning that the user is not 93 valid, or a tuple of (secureToken, username, userRoles). 45 94 """ 46 95 # First check if the user is valid via a cookie 47 96 cookieCheck=self._checkCookie() 48 #o=open('/tmp/tmp/cook.txt','w'); o.write(str(cookieCheck)) ; o.close() 49 50 if type(cookieCheck)==type(""): 51 # Return an error string to report to main application 52 return cookieCheck 97 #o=open('/tmp/cook.txt','w'); o.write(str(cookieCheck)) ; o.close() 98 99 if type(cookieCheck)==type("") or cookieCheck==None: 100 # Didn't get a local cookie, so try and get NDG cookie 101 self.getUsernameAndRolesFromNDGLogin() 102 #o=open('/tmp/co.txt','w'); o.write(str(self.username)) ; o.close() 103 if self.username!=None and self.roles!=None: 104 # Now we have username and roles, make local cookie 105 cookieString=self._createCookie(self.username, self.roles) 106 return (cookieString, self.username, self.roles) 107 else: 108 # If string error then need to try and login again locally 109 return cookieCheck 110 53 111 elif type(cookieCheck)==type([]): 54 112 # Return the valid secure token and user roles 55 113 (cookieString, username, userRoles)=cookieCheck 56 114 return (cookieString, username, userRoles) 57 58 # If no cookie then check if there is a valid username, password provided 59 knownUserPasswords={"rod":"rod1", "jane":"jane1", "freddie":"freddie1", 60 "zippy":"zippy1"} 61 knownUserRoles={"rod":["dset1"], 62 "jane":["dset1", "dset2", "dset3"], 63 "freddie":["dset3"], 64 "zippy":[]} 65 users=knownUserPasswords.keys() 66 67 # Check if username and password given 68 if self.username==None or self.password==None: 69 return "Please login with username and password." 70 71 if self.username in users: 72 if self.password==knownUserPasswords[self.username]: 73 userRoles=knownUserRoles[self.username] 74 #cookieString=":".join(userRoles) 75 cookieString=self._createCookie(self.username, userRoles) 76 return (cookieString, self.username, userRoles) 77 else: 78 return "Invalid login." 79 else: 80 return "Username '%s' unknown." % self.username 81 82 def dummy(self): 83 c=Cookie.SimpleCookie() 84 c["DX"]="somethingOrother" 85 c["DX"]["domain"]="localhost" 86 c["DX"]["path"]="/" 87 print c 115 116 88 117 89 118 def _createCookie(self, username, userRoles, expiryTime=None): … … 91 120 Writes a cookie to the user's browser cookie cache. 92 121 """ 93 self.dummy()94 122 # NOTE: This should be brought up to date with W3C spec on Cookies 95 123 endTime=time.time()+TOKEN_VALID_LIFETIME … … 102 130 103 131 # Use expiry time of zero to delete a cookie, or other time if used 104 if expiryTime !=None:132 if expiryTime==None: 105 133 expiryTime=endTimeString 106 134 … … 149 177 else: 150 178 return "Your log in has expired. Please log in again." 151 152 def _readCookie(self): 153 """ 154 Reads the content of current cookie. 179 180 return "You are not logged in with local login." 181 182 def _readCookie(self, cookie_name=COOKIE_NAME): 183 """ 184 Reads the content of a specified cookie. 155 185 """ 156 186 cookieReader=Cookie.SimpleCookie() … … 161 191 cookieReader.load(os.environ["HTTP_COOKIE"]) 162 192 try: 163 cookieString=cookieReader[ COOKIE_NAME].value193 cookieString=cookieReader[cookie_name].value 164 194 except: 165 195 cookieString=None -
TI03-DataExtractor/trunk/pydxc/__init__.py
r794 r1244 15 15 16 16 # clientConfig.py - all the user-related configuration details 17 from clientConfig import *17 #from clientConfig import * 18 18 19 19 # SecurityViaCGI is the class used to implement security -
TI03-DataExtractor/trunk/pydxc/common.py
r1160 r1244 139 139 140 140 141 def getIdenticalDomains(varLists): 142 """ 143 Returns a list of var IDs that are the on the same domain. 144 """ 145 sameDomains=[] 146 (categories, details)=varLists[0:2] 147 foundDict={} 148 149 for c in range(len(categories)): 150 category=categories[c] 151 detail=details[c] 152 found=None 153 for (cat, det) in foundDict.items(): 154 if detail==det[0]: 155 foundDict[cat].append(category) 156 found=1 157 if found==None: 158 foundDict[category]=[detail] 159 160 codePattn=re.compile(r"axis_(\d+\.\d+\.\d+)\.\d+") 161 for (cat, content) in foundDict.items(): 162 if len(content)>1: 163 axes=[cat]+content[1:] 164 # Now we test for differences in the axes 165 codelist=[codePattn.match(ax).groups()[0] for ax in axes] 166 167 found=[] 168 for code in codelist: 169 if code not in found: found.append(code) 170 171 if len(found)>1: 172 stuffToAdd=[("variable_%s" % f) for f in found] 173 if stuffToAdd not in sameDomains: 174 sameDomains.append(stuffToAdd) 175 176 return sameDomains 177 178 141 179 def deUnicodeObject(obj): 142 180 """ … … 206 244 print getDateTimeComponents("2004-1-1T12:0:0.0") 207 245 print getDateTimeComponents("2004-07-04T12:45:33.489") 246 247 print getIdenticalDomains([["axis_1.3.1.1", "axis_1.1.1.2", "axis_1.1.1.3"], [["lat", None, 32], ["lon", "ough", 45], ["lat", None, 32]]]) 248 249 print getIdenticalDomains([['axis_1.1.1.1', 'axis_1.1.1.2', 'axis_1.1.1.3', 250 'axis_1.1.2.1', 'axis_1.1.2.2', 'axis_1.1.2.3'], [['time', 'time', 'Time', 'hour', 'start end interval', '', '1999-1-1T0:0:0.0', '1999-1-1T6:0:0.0', 6.0], ['latitude', 'latitude', 'Latitude', 'degrees_north', 'start end', '', 90.0, -90.0], ['longitude', 'longitude', 'Longitude', 'degrees_east', 'start end', '', 0, 355], ['time', 'time', 'Time', 'hour', 'start end interval', '', '1999-1-1T0:0:0.0', '1999-1-1T6:0:0.0', 6.0], ['latitude', 'latitude', 'Latitude', 'degrees_north', 'start end', '', 90.0, -90.0], ['longitude', 'longitude', 'Longitude', 'degrees_east', 'start end', '', 0, 355]]]) -
TI03-DataExtractor/trunk/pydxs/CSMLDataHandler.py
r1153 r1244 1 # Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ). 2 # This software may be distributed under the terms of the 3 # Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt 1 # Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ). 2 # This software may be distributed under the terms of the 3 # Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt 4 5 """ 6 CSMLDataHandler.py 7 ================== 8 9 CSMLDataHandler module for the dx package. 10 11 This module holds the CSMLDataHandler class that is used 12 to hold and access information about datasets held in CSML-type 13 formats visible to the dx package. 14 15 """ 16 17 # Import required modules 18 import os 19 import sys 20 sys.path.append("/disks/glue1/astephens") 21 sys.path.append("/disks/glue1/astephens/csml/parser") 22 sys.path.append("/disks/glue1/astephens/csml/Scanner") 23 #import API # csml's api class 24 import re 25 import cdms 26 27 # Import global variables 28 from serverConfig import * 29 from common import * 30 from DateTimeManager import * 31 from DXDMLHandler import DXDMLHandler 32 from DXErrors import * 33 34 # Set up global variables 35 dateTimePattern=re.compile(r"^(\d{4}).(\d{1,2}).(\d{1,2})(\s+|T)(\d+):(\d+):(\d+)\.?(\d+)?") 36 37 38 class CSMLDataHandler: 39 """ 40 A group of methods to connect to a dataset group or 41 dataset to extract information about the contents. 42 """ 43 44 def __init__(self, datasetURI=None): 45 """ 46 Set up instance variables. 47 """ 48 self.DXDML=DXDMLHandler() 49 self.file=datasetURI 50 if self.file: self._openDataFile(datasetURI=self.file) 51 52 53 def _openDataFile(self, datasetGroup=None, dataset=None, datasetURI=None): 54 """ 55 Opens a file and allocates to file handle called: self.file. 56 """ 57 if datasetURI: 58 csmlfile=datasetURI 59 else: 60 for item in self.DXDML.getDatasetsAndDatasetURIs(datasetGroup): 61 if item[0]==dataset: 62 csmlfile=item[1] 63 64 # Check if path starts with "file:" 65 if csmlfile[:5]=="file:": 66 csmlfile=csmlfile[5:] 67 68 try: 69 self.file=API.Parser.Dataset() 70 self.file.parse(csmlfile) 71 except IOError, error: 72 raise DXDataIOError, error 73 74 75 def _getVariable(self, varname): 76 """ 77 Gets variable metadata object from a data file. 78 """ 79 try: 80 rtvalue=self.file.getFeature(varname) 81 except: 82 raise DXOptionHandlingError, "Cannot find variable %s in file %s" % (varname, self.file.id) 83 return rtvalue 84 85 86 def _getBestName(self, v, vcount=0): 87 """ 88 Returns the best name for a cdms variable. 89 """ 90 if not hasattr(v, "standard_name"): 91 if not hasattr(v, "long_name"): 92 if not hasattr(v, "title"): 93 if not hasattr(v, "name"): 94 if hasattr(v, "id"): 95 name=v.id 96 else: 97 vcount=vcount+1 98 name="unnamed_var_%s" % vcount 99 else: 100 name=v.name 101 else: 102 name=v.title 103 else: 104 name=v.long_name 105 else: 106 name=v.standard_name 107 return name 108 109 110 def getVariables(self, datasetGroup=None, dataset=None, datasetURI=None): 111 """ 112 Returns a list of variables for the given dataset 113 group/dataset combination or datasetURI. The variable name used is selected 114 hierarchically depending on the available attributes. Each returned item in 115 the list includes a [<long_name>, <id>]. 116 """ 117 self._openDataFile(datasetGroup, dataset, datasetURI) 118 features=self.file.getFeatureList() 119 rtvars=[] 120 vcount=0 121 122 for ftid in features: 123 v=self.file.getFeature(ftid) 124 name=v.description 125 126 # Fix name to remove leading asterisks and lower case surface start. 127 name=name.replace("_", " ") 128 if name[:2]=="**": name=(name[2:]).strip() 129 if name[:7]=="surface": name=(name[7:]).strip() 130 # Remove variables they are actually bounds on axes or coefficients in formulae 131 if v.id not in ("bounds_longitude", "bounds_latitude", "bounds_level", "bounds_time", "p0"): 132 rtvars.append([name, v.id]) 133 134 return rtvars 135 136 def fuzzyMatch(self, item, matcher): 137 import re 138 match=re.match(r"(%s)(_\d+)?" % matcher, item) 139 if match: return match.groups()[0] 140 return None 141 142 def getDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 143 """ 144 Returns the full domain listing for a variable returning: 145 146 [knownAxisString, id, longName, units, listType, unusedItem, 147 listValue-1, listValue-2, ..., listValue-n] 148 149 For example: 150 151 ["time", "time", "Time", "hours since 1999-09-09 00:00:00", "start end interval", 152 "", 0, 3, 6] 153 154 This listType represents 6-hourly time steps of 0,1,2,3 past the base time. 155 156 listType can also take the value "full list" where all values in the list are provided, 157 or "start end" where only the first and last value are given. 158 """ 159 self._openDataFile(datasetGroup, dataset, datasetURI) 160 var=self._getVariable(variable) 161 varList=self.file.getFeatureList() 162 varList.sort() 163 164 varIndex=varList.index(var.id) 165 rtlist=[] 166 167 axcount=0 168 169 axisList=self._getOrderedAxisList(var) 170 171 for axis in axisList: 172 #axisIndexString="axis%s.%s" % ("77777", axcount) #(varIndex+1, axcount) 173 (name, values)=axis 174 units=None 175 if name=="t": 176 knownAxis="time" 177 (start, end, (intervalValue, intervalUnits))=self.getTemporalDomain(datasetGroup, dataset, variable, datasetURI) 178 arrayValues=[start, end, intervalValue] 179 listType="start end interval" 180 units=intervalUnits 181 elif self.fuzzyMatch(name,"level"): 182 knownAxis="level" 183 arrayValues=values[:] 184 listType="full list" 185 elif self.fuzzyMatch(name,"latitude"): 186 knownAxis="latitude" 187 arrayValues=[values[0], values[-1]] 188 arrayValues.sort() 189 arrayValues.reverse() 190 listType="start end" 191 elif self.fuzzyMatch(name,"longitude"): 192 knownAxis="longitude" 193 arrayValues=[values[0], values[-1]] 194 arrayValues.sort() 195 listType="start end" 196 else: 197 # For any axis not known as above 198 knownAxis="" 199 if len(values[:])>200: 200 arrayValues=[values[0], values[-1]] 201 listType="start end" 202 else: 203 arrayValues=values[:] 204 listType="full list" 205 206 id=name 207 longName=name.title() 208 if not units: units="" 209 210 unused="" 211 newList=[] 212 if type(arrayValues)!=type([]): 213 for i in arrayValues: 214 newList.append(i) 215 arrayValues=newList 216 rtlist.append([knownAxis, id, longName, units, listType, unused]+arrayValues) 217 axcount=axcount+1 218 219 return rtlist 220 221 222 def getHorizontalDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 223 """ 224 Returns the horizontal domain as (northernExtent, westernExtent, southernExtent, easternExtent). 225 """ 226 raise "getHorizontalDomain - has been deprecated!" 227 self._openDataFile(datasetGroup, dataset, datasetURI) 228 var=self._getVariable(variable) 229 lat=list(var.getLatitude()[:]) 230 if lat[-1]<lat[0]: lat.reverse() 231 (southernExtent, northernExtent)=(lat[0], lat[-1]) 232 lon=var.getLongitude()[:] 233 (westernExtent, easternExtent)=(lon[0], lon[-1]) 234 return (northernExtent, westernExtent, southernExtent, easternExtent) 235 236 237 def getVerticalSpatialDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 238 """ 239 Returns the vertical domain as a tuple containing 240 a list of levels (or "Single level" string) and the units. 241 """ 242 raise "getVerticalSpatialDomain - has been deprecated!" 243 self._openDataFile(datasetGroup, dataset, datasetURI) 244 var=self._getVariable(variable) 245 try: 246 levels=var.getLevel() 247 vertical_units=levels.units 248 vertical_domain=tuple(map(lambda x: float(x), levels[:])) 249 250 except: 251 vertical_domain=("Single level",) 252 vertical_units=None 253 return (vertical_domain, vertical_units) 254 255 256 def getTemporalDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 257 """ 258 Returns the temporal domain as a tuple of (start time, end time, 259 (interval value, interval units)). 260 """ 261 self._openDataFile(datasetGroup, dataset, datasetURI) 262 var=self._getVariable(variable) 263 time_axis=var.getDomainReference()["t"].split() 264 #time_keys=("year", "month", "day", "hour", "minute", "second") 265 start_time=str(time_axis[0]).replace(" ", "T") 266 end_time=str(time_axis[-1]).replace(" ", "T") 267 268 time_units="months" 269 if time_units[-1]=="s": time_units=time_units[:-1] 270 271 272 """if len(time_axis)>1: 273 interval_value=abs(time_axis[1]-time_axis[0])""" 274 if len(time_axis)>1: 275 # convert array to relative times 276 #relTimes=[] 277 #relTimeAxis=cdms.createAxis([0]) 278 #relTimeAxis.designateTime() 279 #relTimeAxis.units="%s since %s" % (time_units, start_time.replace("T", " ")) 280 dtp=dateTimePattern 281 match=dtp.match(start_time) 282 if not match: 283 raise "Cannot match date time pattern: %s" % start_time 284 matchlist=match.groups() 285 timeItems=[int(i) for i in (matchlist[:3]+matchlist[4:7])] 286 timeObj=apply(DateTimeManager, tuple(timeItems))#[float[i] for i in match.groups()])) 287 288 matchlist2=dtp.match(time_axis[1]).groups() 289 secondTimeObj=apply(DateTimeManager, tuple([int(x) for x in matchlist2[:3]+matchlist2[4:7]])) 290 secondTimeLong=long(secondTimeObj.formatTime("%Y%m%d%H%M%S")) 291 # Step through times in order to get interval (only works for hours at present). 292 293 for i in range(1,6000000,1): 294 timeObj.add(1, time_units) 295 if long(timeObj.formatTime("%Y%m%d%H%M%S"))==secondTimeLong: 296 interval_value=i 297 break 298 else: 299 interval_value=1 # To stop loops breaking later!""" 300 return (start_time, end_time, (interval_value, time_units)) 301 302 303 def getSelectedTimeSteps(self, datasetURI, variable, axisSelectionDict): 304 """ 305 Returns a list of time step strings based on the selection. 306 """ 307 self._openDataFile(datasetURI=datasetURI) 308 var=self._getVariable(variable) 309 310 timeAxis=None 311 axcount=0 312 313 for axis in self._getOrderedAxisList(var): 314 (name, values)=axis 315 if name=="t": 316 timeAxisIndex=axcount 317 timeAxis=axis 318 axcount=axcount+1 319 320 if timeAxis==None: 321 return [] 322 323 startDateTime=None 324 for key in axisSelectionDict.keys(): 325 #print key, axisSelectionDict[key] 326 axisIndex=int(key.split(".")[-1])-1 327 if axisIndex==timeAxisIndex: 328 (startDateTime, endDateTime)=axisSelectionDict[key][:2] 329 330 # If no time selection specified just return the full list 331 if startDateTime==None: 332 return timeAxis[1] 333 334 startDateTime=startDateTime.replace("T", " ") 335 items=startDateTime.split(":") 336 startDateTime=":".join(items[:-1])+":"+("%f" % float(items[-1])) 337 endDateTime=endDateTime.replace("T", " ") 338 items=endDateTime.split(":") 339 endDateTime=":".join(items[:-1])+":"+("%f" % float(items[-1])) 340 341 timeSteps=timeAxis[1] 342 selectedTimes=[] 343 344 for timeStep in timeSteps: 345 #ts=timeStep 346 match=dateTimePattern.match(timeStep) 347 matchlist=list(match.groups()[:3])+list(match.groups()[4:6]) 348 matchlist=[int(i) for i in matchlist] 349 matchlist=matchlist+[float("%s.%s" % match.groups()[6:8])] 350 ts="%.4d-%.2d-%.2d %.2d:%.2d:%f" % tuple(matchlist) 351 #print str(timeStep), startDateTime 352 353 #ts=str(timeStep).replace("T", " ") 354 #print ts, startDateTime 355 if ts>endDateTime: 356 break 357 elif ts<startDateTime: 358 continue 359 else: 360 selectedTimes.append(ts) 361 362 if selectedTimes==[]: 363 raise DXOptionHandlingError, "All selected time steps for '%s' are out of range, please go back and re-select." % variable 364 365 return selectedTimes 366 367 368 def getSelectedVariableArrayDetails(self, datasetURI, variable, axisSelectionDict): 369 """ 370 Returns a tuple representing the (array shape, grid shape, size) 371 of the selected subset of a variable. Grid shape can be None if both latitude 372 and longitude axes are not present. 373 """ 374 self._openDataFile(datasetURI=datasetURI) 375 var=self._getVariable(variable) 376 varType='d' # Hard-coded for CSML # var.typecode() 377 378 timeAxisIndex=None 379 axcount=0 380 axisList=self._getOrderedAxisList(var) 381 382 for axis in axisList: 383 if axis[0]=="t": 384 timeAxisIndex=axcount 385 axcount=axcount+1 386 387 startDateTime=None 388 389 axesCounted=[] 390 axLens=[] 391 latLength=None 392 lonLength=None 393 size=1 394 for key in axisSelectionDict.keys(): 395 #print key, axisSelectionDict[key] 396 # Set the axisIndex as value minus one because we index from 1 in the dx lists 397 axisIndex=int(key.split(".")[-1])-1 398 (low, high)=axisSelectionDict[key][:2] 399 axis=self._getAxisFromIndex(var, axisIndex) 400 (name, values)=axis 401 402 if axisIndex==timeAxisIndex: 403 axlen=len(self.getSelectedTimeSteps(datasetURI, variable, {key:(low, high)})) 404 elif low==high: 405 axlen=1 406 else: 407 axlen=len(getValuesInRange(low, high, values[:])) 408 if axlen==0: 409 if (self.fuzzyMatch(name,"longitude") or self.fuzzyMatch(name,"latitude")) and low==high: 410 print "Lat and lon can be axis length zero because we'll nudge to nearest if only one value given." 411 axlen=1 412 else: 413 raise DXOptionHandlingError, "All selected '%s' axis values for '%s' are out of range, please go back and re-select." % (name, variable) 414 415 if self.fuzzyMatch(name,"latitude"): 416 latLength=axlen 417 elif self.fuzzyMatch(name,"longitude"): 418 lonLength=axlen 419 420 size=size*axlen 421 axesCounted.append(axisIndex) 422 axLens.append(axlen) 423 424 print axesCounted 425 print axLens 426 427 axcount=0 428 arrayShape=[] 429 for axis in self._getOrderedAxisList(var): 430 (name, values)=axis 431 if axcount not in axesCounted: 432 size=size*len(values) 433 arrayShape.append(len(values)) 434 else: 435 #print axcount 436 arrayShape.append(axLens[axesCounted.index(axcount)]) 437 if self.fuzzyMatch(name,"latitude") and latLength==None: 438 latLength=len(axis) 439 elif self.fuzzyMatch(name,"longitude") and lonLength==None: 440 lonLength=len(axis) 441 axcount=axcount+1 442 443 # Now work out gridShape if appropriate 444 if latLength and lonLength: 445 gridShape=(latLength, lonLength) 446 else: 447 gridShape=None 448 449 if varType=="f": 450 size=size*4. 451 elif varType=="d": 452 size=size*8 453 elif varType=="i": 454 size=size 455 456 return (tuple(arrayShape), gridShape, size) 457 458 459 def getSelectedVariableSubsetSize(self, datasetURI, varID, axisSelectionDict): 460 """ 461 Returns the size in bytes of the selected subset of a variable. 462 """ 463 return self.getSelectedVariableArrayDetails(datasetURI, varID, axisSelectionDict)[2] 464 465 466 # def readVariableSubsetIntoMemory(self, datasetURI, variable, axisSelectionDict, timeStep=None): 467 def subsetVariableToCSMLNC(self, datasetURI, variable, axisSelectionDict, csmlPath, ncPath, timeStep=None): 468 """ 469 Reads the variable with ID 'variable' into memory from file 470 'datasetURI' - sub-setting across all axes indicated in 'axisSelectionDict'. 471 If 'timeStep' is provided then override the time selection in 'axisSelectionDict' 472 with the 'timeStep' given. 473 """ 474 self._openDataFile(datasetURI=datasetURI) 475 var=self._getVariable(variable) 476 477 axisList=self._getOrderedAxisList(var) 478 479 selectionDict={} 480 timeSelection=[] 481 for key in axisSelectionDict.keys(): 482 #print key, axisSelectionDict[key] 483 axisIndex=int(key.split(".")[-1])-1 484 axis=axisList[axisIndex] 485 (name, values)=axis 486 id=name 487 488 # deal with time differently 489 if name=="t": 490 if timeStep!=None: 491 timeStep=timeStep.replace(" ", "T") 492 timeSelection=[self._fixTimeStringForCSML(timeStep)] 493 else: 494 selector=axisSelectionDict[key][:2] 495 selector=(selector[0].replace(" ", "T"), selector[1].replace(" ", "T")) 496 selector=[self._fixTimeStringForCSML(ts) for ts in selector] 497 if selector[0]==selector[1]: selector=[selector[0]] 498 timeSelection=selector 499 elif self.fuzzyMatch(name,"latitude"): 500 (low, high)=axisSelectionDict[key][:2] 501 if low==high: 502 loworig=low 503 (low, high, nudgeMessage)=nudgeSingleValuesToAxisValues(low, values[:], "Latitude") 504 if loworig!=low: print "Nudged latitudes to nearest points..." 505 selectionDict[name]=(low, high) 506 elif self.fuzzyMatch(name,"longitude"): 507 (low, high)=axisSelectionDict[key][:2] 508 if low==high: 509 loworig=low 510 (low, high, nudgeMessage)=nudgeSingleValuesToAxisValues(low, values[:], "Longitude") 511 if loworig!=low: print "Nudged latitudes to nearest points..." 512 selectionDict[name]=(low, high) 513 else: 514 selector=axisSelectionDict[key][:2] 515 selectionDict[name]=selector 516 517 if timeSelection==[]: 518 for axis in axisList: 519 if axis[0]=="t": 520 selector=axis[1] 521 selector=[self._fixTimeStringForCSML(ts) for ts in selector] 522 timeSelection=selector 523 524 print "Generating axis list values for any axes not selected explicitly by user...[NOT IMPLEMENTED]" 525 526 print "timeSelection:", timeSelection 527 print "selectionDict:", selectionDict 528 529 (subsetCSML, subsetNetCDF, arraySize)=var.subsetToGridSeries(timeSelection, csmlPath, ncPath, **selectionDict) 530 #variableData=eval("self.file('%s', %s)" % (variable, fullSelectionString)) 531 return #variableData 532 533 534 def getCFGlobalAttributes(self, datafile): 535 """ 536 Gets any CF metadta global attributes that are available 537 from the source _getOrderedAxisList(self, var)dataset/file. 538 """ 539 # Make sure data file is open 540 if self.file==None: self._openDataFile(datasetURI=datafile) 541 gatts={} 542 543 for gatt in CF_METADATA_GLOBAL_ATTRIBUTE_KEYS: 544 if hasattr(self.file, gatt): 545 gatts[gatt]=self.file.__getattr__(gatt) 546 547 return gatts 548 549 550 def _getOrderedAxisList(self, var): 551 """ 552 For CSML API - returns ordered list of axes. 553 """ 554 axisList=[] 555 dr=var.getDomainReference() 556 drkey=dr.keys()[0] 557 axisList.append([drkey, dr[drkey].split()]) 558 559 dc=var.getDomainComplement() 560 for key in ("level", "latitude", "longitude"): 561 for dckey in dc.keys(): 562 563 if self.fuzzyMatch(dckey, key):# in dc.keys(): 564 axisList.append([dckey, dc[dckey]]) 565 print "Returning from _getOrderedAxisList()" 566 return axisList 567 568 569 def _getAxisFromIndex(self, var, axindex): 570 """ 571 For CSML API - returns axis for var given index. 572 """ 573 return self._getOrderedAxisList(var)[axindex] 574 575 576 def _fixTimeStringForCSML(self, timeString): 577 """ 578 Removes zero padding from time string. 579 """ 580 matchlist=list(dateTimePattern.match(timeString).groups()) 581 if matchlist[-1]==None: matchlist[-1]=0 582 timeItems=[int(i) for i in (matchlist[:3]+matchlist[4:8])] 583 newString="%s-%s-%sT%s:%s:%s.%s" % tuple(timeItems) 584 return newString 585 586 587 if __name__=="__main__": 588 a=CSMLDataHandler() 589 """print a.getVariables(datasetGroup='CSML test dataset group', dataset='CSML test dataset great test') 590 591 print a.getVariables(datasetGroup='CSML test dataset group', datasetURI='file:/usr/local/test/dxs/testdata/csml1.xml') 592 593 print a.getDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 594 595 #print a.getHorizontalDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 596 # These have been deprecated! 597 #print a.getVerticalSpatialDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 598 599 print a.getTemporalDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 600 601 print a.getCFGlobalAttributes("file:/usr/local/test/dxs/testdata/csml1.xml") 602 603 604 print a.getSelectedVariableArrayDetails('file:/usr/local/test/dxs/testdata/csml1.xml', "var2", 605 {"axis_1.1.1.1":("2006-05-15T12:00:00", "2006-06-15T12:00:00"), 606 "axis_1.1.1.2":(10,20), "axis_1.1.1.3":(0,90)}) 607 608 print "Array Details..." 609 x=a.getSelectedVariableArrayDetails('file:/usr/local/test/dxs/testdata/csml1.xml', "var2", 610 {"axis_1.1.1.1":("2006-05-15T12:00:00", "2006-06-15T12:00:00"), 611 "axis_1.1.1.3":(30,-30)}) 612 613 print x 614 615 print "Size..." 616 print x[2] 617 618 print "\n\n\nREALLY SUBSET" 619 print a.subsetVariableToCSMLNC("file:/usr/local/test/dxs/testdata/csml1.xml", "var2", 620 {'axis_1.1.1.1':('2006-05-15T12:00:0.000000', '2006-06-15T12:00:0.000000'), 621 'axis_1.1.1.3': [0.0, 35.0], 'axis_1.1.1.2': [40.0, 90.0]}, "/tmp/a.csml", "/tmp/a.nc") 622 623 print "\nTry another..." 624 print a.subsetVariableToCSMLNC("file:/usr/local/test/dxs/testdata/csml1.xml", "var2", 625 ge {'axis_1.1.1.3': [0.0, 75.0], 'axis_1.1.1.2': [20.0, 90.0], 626 'axis_1.1.1.1':("2006-05-15T12:00:00", "2006-06-15T12:00:00")}, "/tmp/a.csml", "/tmp/a.nc")""" 627 628 print a.subsetVariableToCSMLNC("file:/disks/glue1/astephens/coapec_metadata/COAPEC_500YrRun_wholerun_monthly_atmos.xml", 629 "gproduct", {"axis_1.1.1.1":["2790-01-16T00:00:00", "2790-02-16T00:00:00"], 630 "axis_1.1.1.2":[0.0, 75.0], "axis_1.1.1.3":[0.0, 75.0]}, "/tmp/a.csml", "/tmp/a.nc") 4 631 5 """ 6 CSMLDataHandler.py 7 ================== 8 9 CSMLDataHandler module for the dx package. 10 11 This module holds the CSMLDataHandler class that is used 12 to hold and access information about datasets held in CSML-type 13 formats visible to the dx package. 14 15 """ 16 17 # Import required modules 18 import os 19 import API # csml's api class 20 import re 21 import cdms 22 23 # Import global variables 24 from serverConfig import * 25 from common import * 26 from DateTimeManager import * 27 from DXDMLHandler import DXDMLHandler 28 from DXErrors import * 29 30 # Set up global variables 31 dateTimePattern=re.compile(r"^(\d{4}).(\d{1,2}).(\d{1,2})(\s+|T)(\d+):(\d+):(\d+)\.?(\d+)?") 32 33 34 class CSMLDataHandler: 35 """ 36 A group of methods to connect to a dataset group or 37 dataset to extract information about the contents. 38 """ 39 40 def __init__(self, datasetURI=None): 41 """ 42 Set up instance variables. 43 """ 44 self.DXDML=DXDMLHandler() 45 self.file=datasetURI 46 if self.file: self._openDataFile(datasetURI=self.file) 47 48 49 def _openDataFile(self, datasetGroup=None, dataset=None, datasetURI=None): 50 """ 51 Opens a file and allocates to file handle called: self.file. 52 """ 53 if datasetURI: 54 csmlfile=datasetURI 55 else: 56 for item in self.DXDML.getDatasetsAndDatasetURIs(datasetGroup): 57 if item[0]==dataset: 58 csmlfile=item[1] 59 60 # Check if path starts with "file:" 61 if csmlfile[:5]=="file:": 62 csmlfile=csmlfile[5:] 63 64 try: 65 self.file=API.Parser.Dataset() 66 self.file.parse(csmlfile) 67 except IOError, error: 68 raise DXDataIOError, error 69 70 71 def _getVariable(self, varname): 72 """ 73 Gets variable metadata object from a data file. 74 """ 75 try: 76 rtvalue=self.file.getFeature(varname) 77 except: 78 raise DXOptionHandlingError, "Cannot find variable %s in file %s" % (varname, self.file.id) 79 return rtvalue 80 81 82 def _getBestName(self, v, vcount=0): 83 """ 84 Returns the best name for a cdms variable. 85 """ 86 if not hasattr(v, "standard_name"): 87 if not hasattr(v, "long_name"): 88 if not hasattr(v, "title"): 89 if not hasattr(v, "name"): 90 if hasattr(v, "id"): 91 name=v.id 92 else: 93 vcount=vcount+1 94 name="unnamed_var_%s" % vcount 95 else: 96 name=v.name 97 else: 98 name=v.title 99 else: 100 name=v.long_name 101 else: 102 name=v.standard_name 103 return name 104 105 106 def getVariables(self, datasetGroup=None, dataset=None, datasetURI=None): 107 """ 108 Returns a list of variables for the given dataset 109 group/dataset combination or datasetURI. The variable name used is selected 110 hierarchically depending on the available attributes. Each returned item in 111 the list includes a [<long_name>, <id>]. 112 """ 113 self._openDataFile(datasetGroup, dataset, datasetURI) 114 features=self.file.getFeatureList() 115 rtvars=[] 116 vcount=0 117 118 for ftid in features: 119 v=self.file.getFeature(ftid) 120 name=v.description 121 122 # Fix name to remove leading asterisks and lower case surface start. 123 name=name.replace("_", " ") 124 if name[:2]=="**": name=(name[2:]).strip() 125 if name[:7]=="surface": name=(name[7:]).strip() 126 # Remove variables they are actually bounds on axes or coefficients in formulae 127 if v.id not in ("bounds_longitude", "bounds_latitude", "bounds_level", "bounds_time", "p0"): 128 rtvars.append([name, v.id]) 129 130 return rtvars 131 132 133 def getDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 134 """ 135 Returns the full domain listing for a variable returning: 136 137 [knownAxisString, id, longName, units, listType, unusedItem, 138 listValue-1, listValue-2, ..., listValue-n] 139 140 For example: 141 142 ["time", "time", "Time", "hours since 1999-09-09 00:00:00", "start end interval", 143 "", 0, 3, 6] 144 145 This listType represents 6-hourly time steps of 0,1,2,3 past the base time. 146 147 listType can also take the value "full list" where all values in the list are provided, 148 or "start end" where only the first and last value are given. 149 """ 150 self._openDataFile(datasetGroup, dataset, datasetURI) 151 var=self._getVariable(variable) 152 varList=self.file.getFeatureList() 153 varList.sort() 154 155 varIndex=varList.index(var.id) 156 rtlist=[] 157 158 axcount=0 159 160 axisList=self._getOrderedAxisList(var) 161 """axisList=[] 162 dr=var.getDomainReference() 163 drkey=dr.keys()[0] 164 axisList.append([drkey, dr[drkey].split()]) 165 166 dc=var.getDomainComplement() 167 for key in ("level", "latitude", "longitude"): 168 if key in dc.keys(): 169 axisList.append([key, dc[key]])""" 170 171 for axis in axisList: 172 #axisIndexString="axis%s.%s" % ("77777", axcount) #(varIndex+1, axcount) 173 (name, values)=axis 174 units=None 175 if name=="time": 176 knownAxis="time" 177 (start, end, (intervalValue, intervalUnits))=self.getTemporalDomain(datasetGroup, dataset, variable, datasetURI) 178 arrayValues=[start, end, intervalValue] 179 listType="start end interval" 180 units=intervalUnits 181 elif name=="level": 182 knownAxis="level" 183 arrayValues=values[:] 184 listType="full list" 185 elif name=="latitude": 186 knownAxis="latitude" 187 arrayValues=[values[0], values[-1]] 188 arrayValues.sort() 189 arrayValues.reverse() 190 listType="start end" 191 elif name=="longitude": 192 knownAxis="longitude" 193 arrayValues=[values[0], values[-1]] 194 arrayValues.sort() 195 listType="start end" 196 else: 197 # For any axis not known as above 198 knownAxis="" 199 if len(values[:])>200: 200 arrayValues=[values[0], values[-1]] 201 listType="start end" 202 else: 203 arrayValues=values[:] 204 listType="full list" 205 206 id=name 207 longName=name.title() 208 if not units: units="" 209 210 unused="" 211 rtlist.append([knownAxis, id, longName, units, listType, unused]+arrayValues) 212 axcount=axcount+1 213 214 return rtlist 215 216 217 def getHorizontalDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 218 """ 219 Returns the horizontal domain as (northernExtent, westernExtent, southernExtent, easternExtent). 220 """ 221 raise "getHorizontalDomain - has been deprecated!" 222 self._openDataFile(datasetGroup, dataset, datasetURI) 223 var=self._getVariable(variable) 224 lat=list(var.getLatitude()[:]) 225 if lat[-1]<lat[0]: lat.reverse() 226 (southernExtent, northernExtent)=(lat[0], lat[-1]) 227 lon=var.getLongitude()[:] 228 (westernExtent, easternExtent)=(lon[0], lon[-1]) 229 return (northernExtent, westernExtent, southernExtent, easternExtent) 230 231 232 def getVerticalSpatialDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 233 """ 234 Returns the vertical domain as a tuple containing 235 a list of levels (or "Single level" string) and the units. 236 """ 237 raise "getVerticalSpatialDomain - has been deprecated!" 238 self._openDataFile(datasetGroup, dataset, datasetURI) 239 var=self._getVariable(variable) 240 try: 241 levels=var.getLevel() 242 vertical_units=levels.units 243 vertical_domain=tuple(map(lambda x: float(x), levels[:])) 244 245 except: 246 vertical_domain=("Single level",) 247 vertical_units=None 248 return (vertical_domain, vertical_units) 249 250 251 def getTemporalDomain(self, datasetGroup=None, dataset=None, variable=None, datasetURI=None): 252 """ 253 Returns the temporal domain as a tuple of (start time, end time, 254 (interval value, interval units)). 255 """ 256 self._openDataFile(datasetGroup, dataset, datasetURI) 257 var=self._getVariable(variable) 258 time_axis=var.getDomainReference()["time"].split() 259 #time_keys=("year", "month", "day", "hour", "minute", "second") 260 start_time=str(time_axis[0]).replace(" ", "T") 261 end_time=str(time_axis[-1]).replace(" ", "T") 262 263 time_units="months" 264 if time_units[-1]=="s": time_units=time_units[:-1] 265 266 267 """if len(time_axis)>1: 268 interval_value=abs(time_axis[1]-time_axis[0])""" 269 if len(time_axis)>1: 270 # convert array to relative times 271 #relTimes=[] 272 #relTimeAxis=cdms.createAxis([0]) 273 #relTimeAxis.designateTime() 274 #relTimeAxis.units="%s since %s" % (time_units, start_time.replace("T", " ")) 275 dtp=dateTimePattern 276 match=dtp.match(start_time) 277 if not match: 278 raise "Cannot match date time pattern: %s" % start_time 279 matchlist=match.groups() 280 timeItems=[int(i) for i in (matchlist[:3]+matchlist[4:7])] 281 timeObj=apply(DateTimeManager, tuple(timeItems))#[float[i] for i in match.groups()])) 282 283 matchlist2=dtp.match(time_axis[1]).groups() 284 secondTimeObj=apply(DateTimeManager, tuple([int(x) for x in matchlist2[:3]+matchlist2[4:7]])) 285 secondTimeLong=long(secondTimeObj.formatTime("%Y%m%d%H%M%S")) 286 # Step through times in order to get interval (only works for hours at present). 287 288 for i in range(1,6000000,1): 289 timeObj.add(1, time_units) 290 if long(timeObj.formatTime("%Y%m%d%H%M%S"))==secondTimeLong: 291 interval_value=i 292 break 293 else: 294 interval_value=1 # To stop loops breaking later!""" 295 return (start_time, end_time, (interval_value, time_units)) 296 297 298 def getSelectedTimeSteps(self, datasetURI, variable, axisSelectionDict): 299 """ 300 Returns a list of time step strings based on the selection. 301 """ 302 self._openDataFile(datasetURI=datasetURI) 303 var=self._getVariable(variable) 304 305 timeAxis=None 306 axcount=0 307 308 for axis in self._getOrderedAxisList(var): 309 (name, values)=axis 310 if name=="time": 311 timeAxisIndex=axcount 312 timeAxis=axis 313 axcount=axcount+1 314 315 if timeAxis==None: 316 return [] 317 318 startDateTime=None 319 for key in axisSelectionDict.keys(): 320 #print key, axisSelectionDict[key] 321 axisIndex=int(key.split(".")[-1])-1 322 if axisIndex==timeAxisIndex: 323 (startDateTime, endDateTime)=axisSelectionDict[key][:2] 324 325 # If no time selection specified just return the full list 326 if startDateTime==None: 327 return timeAxis[1] 328 329 startDateTime=startDateTime.replace("T", " ") 330 items=startDateTime.split(":") 331 startDateTime=":".join(items[:-1])+":"+("%f" % float(items[-1])) 332 endDateTime=endDateTime.replace("T", " ") 333 items=endDateTime.split(":") 334 endDateTime=":".join(items[:-1])+":"+("%f" % float(items[-1])) 335 336 timeSteps=timeAxis[1] 337 selectedTimes=[] 338 339 for timeStep in timeSteps: 340 #ts=timeStep 341 match=dateTimePattern.match(timeStep) 342 matchlist=list(match.groups()[:3])+list(match.groups()[4:6]) 343 matchlist=[int(i) for i in matchlist] 344 matchlist=matchlist+[float("%s.%s" % match.groups()[6:8])] 345 ts="%.4d-%.2d-%.2d %.2d:%.2d:%f" % tuple(matchlist) 346 print str(timeStep), startDateTime 347 348 #ts=str(timeStep).replace("T", " ") 349 #print ts, startDateTime 350 if ts>endDateTime: 351 break 352 elif ts<startDateTime: 353 continue 354 else: 355 selectedTimes.append(ts) 356 357 if selectedTimes==[]: 358 raise DXOptionHandlingError, "All selected time steps for '%s' are out of range, please go back and re-select." % variable 359 360 return selectedTimes 361 362 363 def getSelectedVariableArrayDetails(self, datasetURI, variable, axisSelectionDict): 364 """ 365 Returns a tuple representing the (array shape, grid shape, size) 366 of the selected subset of a variable. Grid shape can be None if both latitude 367 and longitude axes are not present. 368 """ 369 self._openDataFile(datasetURI=datasetURI) 370 var=self._getVariable(variable) 371 varType='d' # Hard-coded for CSML # var.typecode() 372 373 timeAxisIndex=None 374 axcount=0 375 axisList=self._getOrderedAxisList(var) 376 377 for axis in axisList: 378 if axis[0]=="time": 379 timeAxisIndex=axcount 380 axcount=axcount+1 381 382 startDateTime=None 383 384 axesCounted=[] 385 axLens=[] 386 latLength=None 387 lonLength=None 388 size=1 389 for key in axisSelectionDict.keys(): 390 #print key, axisSelectionDict[key] 391 # Set the axisIndex as value minus one because we index from 1 in the dx lists 392 axisIndex=int(key.split(".")[-1])-1 393 (low, high)=axisSelectionDict[key][:2] 394 axis=self._getAxisFromIndex(var, axisIndex) 395 (name, values)=axis 396 397 if axisIndex==timeAxisIndex: 398 axlen=len(self.getSelectedTimeSteps(datasetURI, variable, {key:(low, high)})) 399 elif low==high: 400 axlen=1 401 else: 402 axlen=len(getValuesInRange(low, high, values[:])) 403 if axlen==0: 404 if (name=="longitude" or name=="latitude") and low==high: 405 print "Lat and lon can be axis length zero because we'll nudge to nearest if only one value given." 406 axlen=1 407 else: 408 raise DXOptionHandlingError, "All selected '%s' axis values for '%s' are out of range, please go back and re-select." % (name, variable) 409 410 if name=="latitude": 411 latLength=axlen 412 elif name=="longitude": 413 lonLength=axlen 414 415 size=size*axlen 416 axesCounted.append(axisIndex) 417 axLens.append(axlen) 418 419 print axesCounted 420 print axLens 421 422 axcount=0 423 arrayShape=[] 424 for axis in self._getOrderedAxisList(var): 425 (name, values)=axis 426 if axcount not in axesCounted: 427 size=size*len(values) 428 arrayShape.append(len(values)) 429 else: 430 #print axcount 431 arrayShape.append(axLens[axesCounted.index(axcount)]) 432 if name=="latitude" and latLength==None: 433 latLength=len(axis) 434 elif name=="longitude" and lonLength==None: 435 lonLength=len(axis) 436 axcount=axcount+1 437 438 # Now work out gridShape if appropriate 439 if latLength and lonLength: 440 gridShape=(latLength, lonLength) 441 else: 442 gridShape=None 443 444 if varType=="f": 445 size=size*4. 446 elif varType=="d": 447 size=size*8 448 elif varType=="i": 449 size=size 450 451 return (tuple(arrayShape), gridShape, size) 452 453 454 def getSelectedVariableSubsetSize(self, datasetURI, varID, axisSelectionDict): 455 """ 456 Returns the size in bytes of the selected subset of a variable. 457 """ 458 return self.getSelectedVariableArrayDetails(datasetURI, varID, axisSelectionDict)[2] 459 460 461 # def readVariableSubsetIntoMemory(self, datasetURI, variable, axisSelectionDict, timeStep=None): 462 def subsetVariableToCSMLNC(self, datasetURI, variable, axisSelectionDict, csmlPath, ncPath, timeStep=None): 463 """ 464 Reads the variable with ID 'variable' into memory from file 465 'datasetURI' - sub-setting across all axes indicated in 'axisSelectionDict'. 466 If 'timeStep' is provided then override the time selection in 'axisSelectionDict' 467 with the 'timeStep' given. 468 """ 469 self._openDataFile(datasetURI=datasetURI) 470 var=self._getVariable(variable) 471 472 axisList=self._getOrderedAxisList(var) 473 474 selectionDict={} 475 timeSelection=[] 476 for key in axisSelectionDict.keys(): 477 #print key, axisSelectionDict[key] 478 axisIndex=int(key.split(".")[-1])-1 479 axis=axisList[axisIndex] 480 (name, values)=axis 481 id=name 482 483 # deal with time differently 484 if name=="time": 485 if timeStep!=None: 486 timeStep=timeStep.replace(" ", "T") 487 timeSelection=[self._fixTimeStringForCSML(timeStep)] 488 else: 489 selector=axisSelectionDict[key][:2] 490 selector=(selector[0].replace(" ", "T"), selector[1].replace(" ", "T")) 491 selector=[self._fixTimeStringForCSML(ts) for ts in selector] 492 if selector[0]==selector[1]: selector=[selector[0]] 493 timeSelection=selector 494 elif name=="latitude": 495 (low, high)=axisSelectionDict[key][:2] 496 if low==high: 497 loworig=low 498 (low, high, nudgeMessage)=nudgeSingleValuesToAxisValues(low, values[:], "Latitude") 499 if loworig!=low: print "Nudged latitudes to nearest points..." 500 selectionDict[name]=(low, high) 501 elif name=="longitude": 502 (low, high)=axisSelectionDict[key][:2] 503 if low==high: 504 loworig=low 505 (low, high, nudgeMessage)=nudgeSingleValuesToAxisValues(low, values[:], "Longitude") 506 if loworig!=low: print "Nudged latitudes to nearest points..." 507 selectionDict[name]=(low, high) 508 else: 509 selector=axisSelectionDict[key][:2] 510 selectionDict[name]=selector 511 512 if timeSelection==[]: 513 for axis in axisList: 514 if axis[0]=="time": 515 selector=axis[1] 516 selector=[self._fixTimeStringForCSML(ts) for ts in selector] 517 timeSelection=selector 518 519 print "timeSelection:", timeSelection 520 print "selectionDict:", selectionDict 521 522 (subsetCSML, subsetNetCDF, arraySize)=var.subsetToGridSeries(timeSelection, csmlPath, ncPath, **selectionDict) 523 #variableData=eval("self.file('%s', %s)" % (variable, fullSelectionString)) 524 return #variableData 525 526 527 def getCFGlobalAttributes(self, datafile): 528 """ 529 Gets any CF metadta global attributes that are available 530 from the source _getOrderedAxisList(self, var)dataset/file. 531 """ 532 # Make sure data file is open 533 if self.file==None: self._openDataFile(datasetURI=datafile) 534 gatts={} 535 536 for gatt in CF_METADATA_GLOBAL_ATTRIBUTE_KEYS: 537 if hasattr(self.file, gatt): 538 gatts[gatt]=self.file.__getattr__(gatt) 539 540 return gatts 541 542 543 def _getOrderedAxisList(self, var): 544 """ 545 For CSML API - returns ordered list of axes. 546 """ 547 axisList=[] 548 dr=var.getDomainReference() 549 drkey=dr.keys()[0] 550 axisList.append([drkey, dr[drkey].split()]) 551 552 dc=var.getDomainComplement() 553 for key in ("level", "latitude", "longitude"): 554 if key in dc.keys(): 555 axisList.append([key, dc[key]]) 556 return axisList 557 558 559 def _getAxisFromIndex(self, var, axindex): 560 """ 561 For CSML API - returns axis for var given index. 562 """ 563 return self._getOrderedAxisList(var)[axindex] 564 565 566 def _fixTimeStringForCSML(self, timeString): 567 """ 568 Removes zero padding from time string. 569 """ 570 matchlist=list(dateTimePattern.match(timeString).groups()) 571 if matchlist[-1]==None: matchlist[-1]=0 572 timeItems=[int(i) for i in (matchlist[:3]+matchlist[4:8])] 573 newString="%s-%s-%sT%s:%s:%s.%s" % tuple(timeItems) 574 return newString 575 576 577 if __name__=="__main__": 578 a=CSMLDataHandler() 579 print a.getVariables(datasetGroup='CSML test dataset group', dataset='CSML test dataset great test') 580 581 print a.getVariables(datasetGroup='CSML test dataset group', datasetURI='file:/usr/local/test/dxs/testdata/csml1.xml') 582 583 print a.getDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 584 585 #print a.getHorizontalDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 586 # These have been deprecated! 587 #print a.getVerticalSpatialDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 588 589 print a.getTemporalDomain('CSML test dataset group', 'CSML test dataset great test', "var2") 590 591 print a.getCFGlobalAttributes("file:/usr/local/test/dxs/testdata/csml1.xml") 592 593 594 print a.getSelectedVariableArrayDetails('file:/usr/local/test/dxs/testdata/csml1.xml', "var2", 595 {"axis_1.1.1.1":("2006-05-15T12:00:00", "2006-06-15T12:00:00"), 596 "axis_1.1.1.2":(10,20), "axis_1.1.1.3":(0,90)}) 597 598 print "Array Details..." 599 x=a.getSelectedVariableArrayDetails('file:/usr/local/test/dxs/testdata/csml1.xml', "var2", 600 {"axis_1.1.1.1":("2006-05-15T12:00:00", "2006-06-15T12:00:00"), 601 "axis_1.1.1.3":(30,-30)}) 602 603 print x 604 605 print "Size..." 606 print x[2] 607 608 print "\n\n\nREALLY SUBSET" 609 print a.subsetVariableToCSMLNC("file:/usr/local/test/dxs/testdata/csml1.xml", "var2", 610 {'axis_1.1.1.1':('2006-05-15T12:00:0.000000', '2006-06-15T12:00:0.000000'), 611 'axis_1.1.1.3': [0.0, 35.0], 'axis_1.1.1.2': [40.0, 90.0]}, "/tmp/a.csml", "/tmp/a.nc") 612 613 print "\nTry another..." 614 print a.subsetVariableToCSMLNC("file:/usr/local/test/dxs/testdata/csml1.xml", "var2", 615 {'axis_1.1.1.3': [0.0, 75.0], 'axis_1.1.1.2': [20.0, 90.0], 616 'axis_1.1.1.1':("2006-05-15T12:00:00", "2006-06-15T12:00:00")}, "/tmp/a.csml", "/tmp/a.nc") 617 618 632 print a.getDomain(variable="gproduct", datasetURI="file:/disks/glue1/astephens/coapec_metadata/COAPEC_500YrRun_wholerun_monthly_atmos.xml") 633 -
TI03-DataExtractor/trunk/pydxs/DXDMLHandler.py
r1225 r1244 138 138 return i[2] 139 139 raise DXOptionHandlingError, "Cannot match any dataset group to the datasetURI: '%s'" % datasetURI 140 141 142 def getDatasetGroupAndDatasetFromURI(self, datasetURI): 143 """ 144 Gets the id of the dataset group and dataset from a 145 datasetURI (if it is known to the dx). 146 """ 147 URI_list=self.getDatasetURIList() 148 for i in URI_list: 149 print i 150 if i[0]==datasetURI or i[0]==("file:"+datasetURI): 151 return (i[2], i[1]) 152 raise DXOptionHandlingError, "Cannot match any dataset group and dataset to the datasetURI: '%s'" % datasetURI 140 153 141 154 -
TI03-DataExtractor/trunk/pydxs/OptionHandler.py
r1225 r1244 65 65 optsRequested="dataset" # to ensure all options are at this level 66 66 elif TOP_LEVEL=="datasetGroup": 67 optsRequested="datasetGroup" # to ensure all options are at this level 67 # What if you only have datasetURIs? 68 if keyPatternMatch(self.bag, "datasetURI_\d+", "regex"): 69 dsuriDict=getDictSubsetMatching(self.bag, "datasetURI_\d+", "regex") 70 71 dsgBag={} 72 for key,dsuri in dsuriDict.items(): 73 n=int(key.split("_")[-1]) 74 (dsg, ds)=self.DXDML.getDatasetGroupAndDatasetFromURI(dsuri) 75 if not dsgBag.has_key(n): 76 dsgBag[n]={"dsg":dsg, "ds":[]} 77 dsgBag[n]["ds"].append(ds) 78 79 # Now bag is full create session selection entries for dataset groups 80 # and datasets to fill the gaps 81 for key,value in dsgBag.items(): 82 self.bag["datasetGroup_%s" % key]=value["dsg"] 83 dscount=1 84 for ds in value["ds"]: 85 self.bag["dataset_%s.%s" % (key, dscount)]=ds 86 optsRequested="variable" # Since we've now populated dsg and ds from dsuri 87 88 else: 89 optsRequested="datasetGroup" # to ensure all options are at this level 68 90 69 91 elif not keyPatternMatch(self.bag, "dataset_\d+\.\d+", "regex"): … … 101 123 if optsRequested=="datasetGroup": 102 124 count=1 103 for dsg in self.getDatasetGroupList(): 125 dsgList=self.getDatasetGroupList() 126 127 for dsg in dsgList: 104 128 choices.append(("datasetGroup_%s" % count, dsg)) 105 129 count=count+1 … … 278 302 Method to return list of available variables for the given datasetGroup 279 303 and dataset/datasetURI. 280 """ 304 """ 281 305 dataHandler=DatasetFormatDecider(datasetGroup, dataset, datasetURI).datasetFormat 282 306 -
TI03-DataExtractor/trunk/pydxs/common.py
r1160 r1244 222 222 else: 223 223 newpath=uri.replace(OUTPUT_DIR, OUTPUT_DIR_URL) 224 print "\n\n", uri, "\n\n", newpath, "\n\n" 224 225 return newpath 225 226 -
TI03-DataExtractor/trunk/setup.py
r1225 r1244 165 165 os.system("rm -fr %s" % target) 166 166 167 167 168 if os.path.isfile(sourceCopy): 168 169 shutil.copy(sourceCopy, target) 169 170 elif os.path.isdir(sourceCopy): 170 shutil.copytree(sourceCopy, target) 171 try: 172 shutil.copytree(sourceCopy, target) 173 except: 174 makeDirAndPerm(os.path.split(target)[0]) 175 shutil.copytree(sourceCopy, target) 171 176 172 177 print "Installed non-python package and scripts under:", BASEDIR … … 188 193 makeDirAndPerm(OUTPUT_DIR_LOCAL_PATH) 189 194 195 # Define config files here so that dxui cgi script can have 196 # the config file location inserted... 197 configMap={"s":os.path.join(BASEDIR, "configs", "serverConfig.py"), 198 "c":os.path.join(BASEDIR, "configs", "clientConfig.py")} 199 200 configFileName=configMap[package[-1]] 201 190 202 if "CGI_SCRIPT_LOCAL_PATH" in configVarDict.keys(): 191 203 try: 204 print "\nAdding your python and location of config file to cgi script:", CGI_SCRIPT_LOCAL_PATH 192 205 shutil.copy("../cgi/dxui", CGI_SCRIPT_LOCAL_PATH) 193 206 os.chmod(CGI_SCRIPT_LOCAL_PATH, 0755) … … 195 208 print thisPython 196 209 os.system("perl -p -i -w -e 's:<YOUR_PYTHON_HERE>:%s:g;' %s*" % (thisPython, CGI_SCRIPT_LOCAL_PATH)) 210 os.system("perl -p -i -w -e 's:<PLACE_CONFIG_PATH_HERE>:%s:g;' %s*" % (configFileName, CGI_SCRIPT_LOCAL_PATH)) 197 211 198 212 print "\nCopied CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH … … 231 245 else: 232 246 makeDirAndPerm(SESSION_OBJECT_DIR, 0777) 233 234 if "MAP_APPLET_TEMPLATE_LOCAL_PATH" in configVarDict.keys(): 235 print "\nChanging permissions on maps/ directory..." 236 os.chmod(os.path.split(MAP_APPLET_TEMPLATE_LOCAL_PATH)[0], 0777) 237 238 configMap={"s":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "serverConfig.py"), 239 "c":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "clientConfig.py")} 240 print "Renaming configuration variables in config file:" 241 242 configFileName=configMap[package[-1]] 247 248 print "Renaming configuration variables in config file:", configFileName 249 243 250 configFile=open(configFileName) 244 251 configLines=configFile.readlines() … … 248 255 249 256 newLines=[] 257 confFileVarDict={} 250 258 for line in configLines: 251 259 matched=None … … 254 262 if re.match(r"\s*%s\s*=" % confVar, line): 255 263 newLine="%s=%s\n" % (confVar, value) 264 confFileVarDict[confVar]=value 256 265 print "CHANGING: %sTO: %s" % (line, newLine) 257 266 newLines.append(newLine) … … 267 276 configFile.close() 268 277 print "Re-written config file: %s\n" % (configFileName) 278 279 280 if "MAP_APPLET_TEMPLATE_LOCAL_PATH" in confFileVarDict.keys(): 281 print "\nChanging permissions on maps/ directory..." 282 os.chmod(os.path.split(MAP_APPLET_TEMPLATE_LOCAL_PATH)[0], 0777) 283 284 # Need my cleaning old python installations code here... 269 285 270 286 if package=="dxs":
Note: See TracChangeset
for help on using the changeset viewer.