source: TI03-DataExtractor/trunk/dxs/bin/DXWSInterface.py @ 794

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/dxs/bin/DXWSInterface.py@794
Revision 794, 21.1 KB checked in by astephen, 13 years ago (diff)

Unstable but latest version with multi-variable support and split hooks
for CDML and CSML.

  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2
3#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
4#   This software may be distributed under the terms of the
5#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
6
7"""
8DXWSInterface.py
9================
10
11A web service wrapper around the entire package.
12
13Wraps the package functionality into the following Web Service methods:
14
15- callControllerDirectly(args)
16
17- startSession(username=None, password=None, secureToken=None)
18
19- getOptions(sessionID, secureToken=None)
20
21- selectOptions(sessionID, opt_1, opt_2, ...,opt_n, secureToken=None)
22
23- isComplete(sessionID, secureToken=None)
24
25- createOutput(sessionID, secureToken=None)
26
27- newSession(sessionID, secureToken=None)
28
29
30Notes about the sessionID and secureToken object
31------------------------------------------------
32
33The sessionID carries information about the current session but has no security
34implications. It is simply a way for the server to keep track of the request
35between calls.
36
37The secureToken is a security object (which should be encoded) that is time limited and provides
38information on what the user is allowed to access on the server. Some servers
39do not run security and so just send around a None type as the secureToken. Those
40implementing security will require that the secureToken (held in a string) is sent
41with each method call. During each call it will updated (to ensure it remains
42time valid) and returned.
43
44All errors will be present in the instance object's self.error variable. Check for
45this after each call to test success.
46
47Notes on ZSI implementation. All objects needed to be wrapped in Python
48lists in order to be returned to the client. Hence a tuple of 4,5 has to
49be represented as [[4],[5]]. Not sure why...
50
51############# THE FOLLOWING MIGHT BE IMPLEMENTED IN FUTURE #############
52getDatasetGroupOptions(sessionID, secureToken=None)
53
54selectDatasetGroups(sessionID, datasetGroup_1, datasetGroup_2, ..., datasetGroup_n, secureToken=None)
55
56getDatasetOptions(sessionID, secureToken=None)
57
58selectDatasets(sessionID, dataset_1, dataset_2, ..., dataset_n, secureToken=None)
59
60getVariableOptions(sessionID, secureToken=None)
61
62selectVariables(sessionID, variable_1, variable_2, ..., variable_n, secureToken=None)
63
64getHorizontalDomainOptions(sessionID, secureToken=None)
65
66selectHorizontalDomain(sessionID, northernExtent, westernExtent,
67       easternExtent, southernExtent, secureToken=None)
68       
69getVerticalDomainOptions(sessionID, secureToken=None)
70
71selectVerticalDomain(sessionID, verticalDomain_1, verticalDomain_2, ...,
72                        verticalDomain_n, secureToken=None)
73                       
74getTemporalDomainOptions(sessionID, secureToken=None)
75
76selectTemporalDomainOptions(sessionID, startTime_1, endTime_1, startTime_2, endTime_2,
77                                ..., startTime_n, endTime_n, secureToken=None)
78                # Note that times are expressed as: "YYYY-MM-DD hh:mm:ss" or minimally as "YYYY"
79                # which convert to "YYYY-01-01 00:00:00".
80               
81getOutputFormatOptions(sessionID, secureToken=None)
82
83selectOutputFormat(sessionID, outputFormat, secureToken=None)
84
85"""
86
87# Import standard library modules
88import sys
89import re
90import urllib
91import os, types
92
93# Import package modules
94from pydxs.common import *
95from pydxs.DXController import *
96from pydxs.serverConfig import *
97from pydxs.DXRMLHandler import *
98from pydxs.DXErrors import *
99
100# Set global variables
101sessionID=None
102timePattern=re.compile("(\d{4})-(\d{1,2})-(\d{1,2}).(\d{1,2}):(\d{1,2}):(\d{1,2}(\.\d+)?)")
103
104 
105def callControllerDirectly(argList):
106    """
107    Function wrapper to Controller class. This can be called directly
108    to perform all operations or the other Web Service methods can be
109    called in the appropriate order.
110    """   
111    # Parse startTime and endTime arguments into expected form
112    for key, value in argList:
113        args[key]=value
114        for ts in ("start", "end"):
115            if key.find("%sTime_" % ts)>-1:
116                timeComponents=time.strp(value, "%Y-%m-%d %H:%M:%S")[:6]
117       
118                for tk in TIME_KEYS:
119                    # Set each component e.g. startYear_1, endHour_2
120                    args["%s%s_%s" % (ts, tk, key.split("_")[-1])]=timeComponents[TIME_KEYS.index(tk)]
121
122    # Parse horizontalDomain argumnets into expected form
123    if args.has_key("horizontalDomain"):
124        for hk in HORIZ_KEYS:
125            args[hk]=args["horizontalDomain"][HORIZ_KEYS.index(hk)]
126                       
127    return Controller(args)
128
129
130
131def startSession(username=None, password=None, secureToken=None):
132    """
133    Logs a user in and supplies them with a session ID as well
134    as an encoded security token. The session ID connects them to their
135    current request serverside whilst the security token is used to
136    authorise their access.
137    """
138    print "Method called: startSession\n"
139    args={"username":username, "password":password, "secureToken":secureToken} 
140       
141    try:
142        controller=DXController(args)
143    except Exception, error:
144        return str(error)           
145       
146    sessionID=controller.bag["sessionID"]
147    secureToken=controller.secureToken
148    # Have to return list of lists for ZSI to work (?!)
149    return [[sessionID], [secureToken]]
150       
151
152def getOptions(sessionID, secureToken=None, optionCategoryRequested=None):
153    """
154    Returns a category description of the next set of options (or the option category
155    requested by the user), a list of options based
156    on what the user has yet to request, a string explaining some more about this and a
157    security token.
158   
159    The options will be presented following the heirarchy:
160        DatasetGroups
161        Datasets
162        Variables
163        HorizontalDomain
164        VerticalDomain
165        TemporalDomain
166        OutputFormat
167    """   
168    print "Method called: getOptions\n"
169    print "ARGS: sessionID: %s\nsecureToken: %s, optionCategoryRequested:%s" % (sessionID, 
170              secureToken, optionCategoryRequested)
171    if type(sessionID)==type(u""):
172        sessionID=str(sessionID)
173    args={"sessionID":sessionID, "secureToken":secureToken, "optionCategoryRequested":optionCategoryRequested}
174
175    try:
176        controller=DXController(args)
177    except Exception, error:
178        return str(error)       
179
180    optionsObject=controller.options
181
182    if optionsObject=={}:
183        print "All options selected..."
184        return  [["No category"],["All options selected"],["All options selected"],[secureToken]]
185
186    ndatasets=len(optionsObject.keys())   
187    # Create a list of option Categories
188    optionCategories=[str(optionsObject[str(n)][0]) for n in range(1, ndatasets+1)] 
189    # Create a list of option lists
190    # If options are lists then need to make l[0] the optionString and
191    # l[1] the option
192
193    options=[optionsObject[str(n)][1:] for n in range(1, ndatasets+1)]
194
195    options2=[]
196    count=0
197    for l in options:
198        options2.append([])
199        for i in l:
200            options2[count].append(deUnicodeObject(i))
201        count=count+1
202 
203    options=options2
204    optionStrings=options
205    if len(optionCategories)>0:
206        optcat0=optionCategories[0].split("_")[0]
207
208        if optcat0=="horizontalDomain" or optcat0=="outputFormat":
209            options=options[0][0]
210        elif optcat0 in ("dataset", "verticalDomain", "temporalDomain"):
211            newOpts=[]
212            optionStrings=[]
213            for opt in options:
214                if type(opt[0][0])==type([]) and optcat0!="temporalDomain":
215                    newOpts.append([opt[0][0][1]])
216                    optionStrings.append([opt[0][0][0]])
217                else:
218                    newOpts.append(opt[0])
219                    optionStrings.append(opt[0])
220
221            options=newOpts
222           
223        elif optcat0=="datasetGroup":
224            # Need to sort out the metadata links sent as [longName, [detailedMetadataLink, discoveryMetadataLink]]
225            newOpts=[]
226            optionStrings=[]
227            for n in range(len(options)):
228                newOpts.append([])
229                optionStrings.append([])
230                               
231            for opt in options[0]:
232                for i in opt:               
233                    newOpts[options[0].index(opt)].append(i)
234                    optionStrings[options[0].index(opt)].append(i[0])
235                   
236            optionStringsMulti=[]
237            optionsMulti=[]
238            for n in range(len(options)):
239                optionStringsMulti.append(optionStrings[0])
240                optionsMulti.append(newOpts[0])
241               
242            optionStrings=optionStringsMulti
243            options=optionsMulti           
244
245        elif optcat0=="variable":
246            newOpts=[]
247            optionStrings=[]
248            for n in range(len(options)):
249                newOpts.append([])
250                optionStrings.append([])
251                               
252            """for opt in options:
253                for i in opt[0]:
254                    newOpts[options.index(opt)].append(i[1])
255                    optionStrings[options.index(opt)].append(i[0])
256       
257            options=newOpts"""
258           
259            counter=0
260            for opt in options:
261                #print "opt:%s\n" % opt
262                for i in opt[0]:
263                    #print "i:",i,"\n"
264                    newOpts[counter].append(i[1])
265                    #print "newOpts:", newOpts, "\n"
266                    optionStrings[counter].append(i[0])
267                counter=counter+1
268           
269            options=newOpts                 
270             
271    print "\nOPTIONS\n==========\n", options, "\n", "OPTIONSTRINGS:\n=================\n", optionStrings, "\nOPTIONCATS:\n================\n",  optionCategories
272
273    secureToken=controller.secureToken   
274   
275    # Returns the optionCategories as the first item for each dataset in order
276    # and a list of lists of options for the corresponding categories as the
277    # second item in the tuple.
278    return [[optionCategories], [options], [optionStrings], [secureToken]]   
279
280
281def selectOptions(sessionID, argList):
282    """
283    Makes a selection based on user requirements. This will typically follow a call
284    to getOptions() to find out what the options are.
285    Returns a call to getOptions() to grab the next set of options available to the
286    user. Alternatively, if it fails, an errorString is returned.
287    """
288    print "Method called: selectOptions\n"   
289    args={}
290    for key,value in argList:
291        key=deUnicodeObject(key)
292        value=deUnicodeObject(value)
293        if type(key)==type(u""):
294            newkey=str(key)
295        else:
296            newkey=key
297        if type(value)==type(u""):
298            value=str(value)
299        args[newkey]=value   
300        print newkey, value
301             
302    try:
303        args["sessionID"]=sessionID
304       
305        if args.has_key("horizontalDomain"):
306            for hkey in HORIZ_KEYS:
307                args[hkey]=args["horizontalDomain"][HORIZ_KEYS.index(hkey)]
308            del args["horizontalDomain"]
309       
310        # Step through as many datasets as the user is choosing
311        for n in range(1, MAX_NUM_DATASETS+1):
312            if args.has_key("temporalDomain_%s" % n):
313                temp=args["temporalDomain_%s" % n]
314               
315                if type(temp[0]) in (types.TupleType, types.ListType, types.InstanceType):
316                    startTimeList=temp[0]
317                    endTimeList=temp[1]     
318                    print 
319                elif type(temp[0])==types.StringType:
320                    startTimeList=[int(i) for i in timePattern.match(temp[0]).groups()[0:6]]
321                    endTimeList=[int(i) for i in timePattern.match(temp[1]).groups()[0:6]]
322
323                else:
324                    raise "Could not identify temporalDomain argument: "+str(temp)   
325
326                startTimeString="%.4d-%.2d-%.2dT%.2d:%.2d:%.2f" % tuple(startTimeList)
327                endTimeString="%.4d-%.2d-%.2dT%.2d:%.2d:%.2f" % tuple(endTimeList)   
328               
329                timeDict={"startTime":startTimeString, "endTime":endTimeString}                       
330                args["startDateTime_%s" % n]=timeDict["startTime"]
331                args["endDateTime_%s" % n]=timeDict["endTime"]
332                args["timeIntervalValue_%s" % n]=temp[2][0]
333                args["timeIntervalUnits_%s" % n]=temp[2][1]
334                del args["temporalDomain_%s" % n]
335               
336            elif args.has_key("verticalDomain_%s" % n):
337                vert=args["verticalDomain_%s" % n]
338                if type(vert) in (types.ListType, types.TupleType) and len(vert)==1:
339                        args["verticalDomain_%s" % n]=(vert[0],)
340                else:
341                        args["verticalDomain_%s" % n]=vert
342       
343        try:
344            controller=DXController(args)
345        except Exception, error:
346            return str(error)
347
348        secureToken=controller.secureToken     
349        if args.has_key("optionCategoryRequested"):
350            optionCategoryRequested=args["optionCategoryRequested"]
351        else:
352            optionCategoryRequested=None
353
354        opts=getOptions(sessionID, secureToken, optionCategoryRequested)
355        return opts
356
357    except:
358        return ("ERROR: Could not make the selection you required.\n"+str(sys.exc_type), secureToken)
359       
360   
361def isComplete(sessionID, secureToken=None):
362    """print
363    Returns 1 if the request is complete (i.e. ready to create output file(s))
364    and 0 if not. Also returns a security token.
365    """
366    print "Method called: isComplete\n"
367    optionsObject=getOptions(sessionID, secureToken)
368    optionCategories=optionsObject[0][0]
369    secureToken=optionsObject[3][0]
370
371    if optionCategories=="No category":
372        return [[1], [secureToken]]
373    else:
374        return [[0], [secureToken]]
375   
376
377def createOutput(sessionID, secureToken=None):
378    """
379    Creates the outputs specified by user selections. It writes these files
380    locally and returns a list of paths to the data as either FTP or HTTP
381    locations.
382    Returns a list of paths and a security token.
383    """
384    print "Method called: createOutput\n"
385    args={"sessionID":sessionID, "secureToken":secureToken, "getOutput":"getOutput"}
386     
387    try:
388        controller=DXController(args)
389    except Exception, error:
390        return str(error)
391   
392    secureToken=controller.secureToken
393    print controller.bag
394    pathList=[controller.bag["outputFilePaths"]]
395    return [[pathList], [secureToken]]
396
397
398def newSession(sessionID, secureToken=None):
399    """
400    Deletes content of current request.
401    Returns the sessionID and the security token..
402    """
403    print "Method called: newSession\n"   
404    args={"sessionID":sessionID, "secureToken":secureToken, "clearSession":"clearSession"}
405     
406    try:
407        controller=DXController(args)
408    except Exception, error:
409        return str(error) 
410   
411    secureToken=controller.secureToken
412    return [[sessionID], [secureToken]]
413
414
415def setNumberOfDatasets(sessionID, n, secureToken=None):
416    """
417    Re-selects the number of datasets the user wants to deal with in the request.
418    Default is 1.
419    Returns 1 if successful and an errorString if not, as well as a security token.
420    """
421    print "Method called: setNumberOfDatasets\n"
422   
423    args={"sessionID":sessionID, "numberOfDatasets":n, "secureToken":secureToken}
424
425    try:
426        controller=DXController(args)
427    except Exception, error:
428        return str(error)   
429
430    secureToken=controller.secureToken
431    return [[1], [secureToken]]
432       
433       
434def summariseRequest(sessionID, secureToken=None):
435    """
436    Returns a listing of the current request in a string and a security token.
437    """   
438    print "Method called: summariseRequest\n"
439    args={"sessionID":sessionID, "secureToken":secureToken}
440   
441    try:
442        controller=DXController(args)
443    except Exception, error:
444        return str(error)   
445
446    req=controller.bag
447    from pydxs.RequestSizer import RequestSizer
448    try:
449        sizer=RequestSizer(req)
450        req["numberOfDataPoints"]=sizer.npoints
451        req["sizeInBytes"]=sizer.size
452    except:
453        pass
454   
455    summaryString="\n"+"*"*40
456    summaryString=summaryString+"\nSummary of current request follows\n"
457    summaryString=summaryString+"*"*40+"\n\n" 
458    orderedSortedKeys=[]
459
460    keybins=[[]]*(MAX_NUM_DATASETS+1)
461    for key in req.keys():
462        if key[0]=="_": continue
463        items=key.split("_")
464        if len(items)==1:  keybins[0].append(key)
465       
466    for key in req.keys():
467        if key[0]=="_": continue
468        items=key.split("_")
469        if len(items)==2:
470            keybins[int(items[1])].append(key)
471               
472    for n in range(0, MAX_NUM_DATASETS+1):
473        keylist=keybins[n]
474        keylist.sort()
475        orderedSortedKeys=orderedSortedKeys+keylist
476   
477    exclusions=("callMethod", "secureToken", "optionCategoryRequested", 
478                "action", "targetPage")
479    for key in keylist:
480        if req[key]!="" and key not in exclusions:  summaryString=summaryString+("%s:\t%s\n" % (key, req[key]))
481
482    secureToken=controller.secureToken 
483    return [[summaryString], [secureToken]]
484     
485
486def uploadRequest(sessionID, requestXMLString, secureToken=None):
487    """
488    Allows the user to send a request in the form of a string containing the
489    contents of an XML file conforming to the dx request specification (not
490    yet written). This is also called a Data Subset Specifier.
491    This function parses the XML into a dictionary and then uploads the arguments.
492    Returns a status flag (1=success, None=failure) and the secure token.
493    """
494    print "Calling: uploadRequest\n"
495   
496    try:
497        args=DXRMLParser(requestXMLString).getDictionary()
498    except DXError, error:
499        return "Error parsing the Request XML file you provided:   \n"+str(error)       
500    except Exception, error:
501        return "Error parsing the Request XML file you provided:   \n"+str(error)
502       
503    args["sessionID"]=sessionID
504    args["secureToken"]=secureToken
505   
506    try:
507        controller=DXController(args)
508    except Exception, error:
509        return [[0], [secureToken]]   
510             
511    secureToken=controller.secureToken 
512    return [[1], [secureToken]]
513
514
515def getDataSubsetSpecifier(sessionID, secureToken=None):
516    """
517    Returns the dataSubsetSpecifier XML document (which might be S-metadata)
518    required by a Delivery Service to describe the subset requested, and a
519    security token.
520    """
521    print "Method called: getDataSubsetSpecifier\n"
522    args={"sessionID":sessionID, "secureToken":secureToken}
523   
524    try:
525        controller=DXController(args)
526    except Exception, error:
527        return str(error)     
528   
529    secureToken=controller.secureToken
530    sessionObj=controller.bag
531   
532    try:
533        dataSubsetSpecifierXMLString=DXRMLGenerator(sessionObj).xmlString
534    except Exception, error:
535        return "Could not provide the Request XML string:   "+str(error)
536   
537    print dataSubsetSpecifierXMLStringprint
538    return [[dataSubsetSpecifierXMLString], [secureToken]]
539
540
541def getExtractionCosts(sessionID, secureToken=None):
542    """
543    Returns an estimated duration for the creation of the output data and the
544    estimated volume of the output.
545    """
546    print "Method called: getExtractionCosts\n"   
547    args={"sessionID":sessionID, "secureToken":secureToken, "action":"requestCosts"}
548   
549    try:
550        controller=DXController(args)
551    except Exception, error:
552        return str(error)   
553
554           
555    secureToken=controller.secureToken
556    (estimatedDuration, estimatedVolume)=(controller.estimatedDuration, controller.estimatedVolume)
557    print "Returning...", [[estimatedDuration], [estimatedVolume], [secureToken]]
558    return [[estimatedDuration], [estimatedVolume], [secureToken]]       
559
560
561####def getOptions(sessionID, secureToken=None, optionCategoryRequested=None):
562    """
563    Returns a category description of the next set of options (or the option category
564    requested by the user), a list of options based
565    on what the user has yet to request, a string explaining some more about this and a
566    security token.
567   
568    The options will be presented following the heirarchy:
569        DatasetGroups
570        Datasets
571        Variables
572        HorizontalDomain
573        VerticalDomain
574        TemporalDomain
575        OutputFormat
576    """   
577
578def getDatasetGroupOptions(sessionID, secureToken=None):
579    """
580    Get the dataset group options available.
581    """
582    print "Method called: getDatasetGroupOptions\n\n"
583    return getOptions(sessionID, secureToken=None, optionCategoryRequested="datasetGroup")
584   
585def makeDatasetGroupSelections(sessionID, secureToken, datasetGroupList):
586    print "Called: makeDatasetGroupSelections\n"
587    argList=[]
588    argList.append(["secureToken", secureToken])
589    counter=1
590    for selection in datasetGroupList:
591        argList.append(["datasetGroup_%s" % counter, selection])
592        counter=counter+1
593       
594    return selectOptions(sessionID, argList)
595
596def getDatasetOptions(sessionID, secureToken=None):
597    """
598    Get the dataset options available.
599    """
600    print "Method called: getDatasetOptions\n\n"
601    return getOptions(sessionID, secureToken=None, optionCategoryRequested="dataset")
602   
603def makeDatasetSelections(sessionID, secureToken, datasetList):
604    print "Called: makeDatasetSelections\n"
605    argList=[]
606    argList.append(["secureToken", secureToken])
607    counter=1
608    for selection in datasetList:
609        argList.append(["dataset_%s" % counter, selection])
610        counter=counter+1
611       
612    return selectOptions(sessionID, argList)
613
614def getVariableOptions(sessionID, secureToken=None):
615    """
616    Get the variable options available.
617    """
618    print "Method called: getVariableOptions\n\n"
619    return getOptions(sessionID, secureToken=None, optionCategoryRequested="variable")
620   
621def makeVariableSelections(sessionID, secureToken, variableList):
622    print "Called: makeVariableSelections\n"
623    argList=[]
624    argList.append(["secureToken", secureToken])
625    counter=1
626    for selection in variableList:
627        argList.append(["variable_%s" % counter, selection])
628        counter=counter+1
629       
630    return selectOptions(sessionID, argList)
631   
632
633if __name__=="__main__": 
634    """
635    Serve all functions as Web Service methods.
636    """
637    print "Importing SOAP library (ZSI)."
638    from ZSI import dispatch
639    print "Setting up server"
640    portNumber=SOAP_SERVER_PORT
641    print "Serving Web Service on port: %s" % portNumber   
642    dispatch.AsServer(port=portNumber)   
643
644
645
Note: See TracBrowser for help on using the repository browser.