source: cows_wps/trunk/cows_wps/process_handler/process_support.py @ 7016

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/cows_wps/trunk/cows_wps/process_handler/process_support.py@7016
Revision 7016, 5.7 KB checked in by astephen, 10 years ago (diff)

fixes to use common finishProcess call

Line 
1import time
2import sys
3import os
4import stat
5import logging
6from datetime import datetime
7
8from cows_wps.utils import metadata_utils
9from cows_wps.process_handler.fileset import FileSet, zipFileSet, dryRunZipFileSet, FLAG
10from cows_wps.process_handler.context.process_status import STATUS
11
12
13log = logging.getLogger(__name__)
14
15def finishProcess(context, fileSet, startTime, 
16                  finalMessage = 'The End', keep = False):
17    """
18    Performs the common finishing up tasks for a processes __call__ method.
19    """
20    writeMetadataFile(context, requestMetadata)
21    processOutputs(context, fileSet, dryRun = False, keep = keep)
22    context.setStatus(STATUS.COMPLETED, finalMessage, 100)
23
24    completionTime = time.time()   
25    updateJobDetailsAfterCompletion(context, startTime, completionTime)
26
27def finishDryRun(context, requestMetadata, fileSet, duration, 
28                 acceptedMessage = "Process execution request has been accepted"):
29    """
30    Performs the common finishing up tasks for a processes dryRun method.
31    """
32
33    writeMetadataFile(context, requestMetadata, dryRun = True)
34    processOutputs(context, fileSet, dryRun = True)
35    context.setStatus(STATUS.ACCEPTED, acceptedMessage , 0)
36
37    updateJobDetailsAfterDryRun(context, duration)
38
39
40def makeMetadataFileItems(context, excludedMetadataInputs, cachable):
41    """
42    Creates the metadata items for a request and adds the 'ReUseURL' item on
43    the end. If cachable is False the value of the 'ReUseURL item will be a
44    warning.
45    """
46       
47    metadataItems = makeMetadataItemsFromInputs(context, excludedMetadataInputs)
48           
49    return metadataItems
50
51def makeMetadataItemsFromInputs(context, excludedMetadataInputs):
52    """
53    Creates a list of (name, value) metadata items from the request's inputs.
54    If an input item appears in the excludedMetadataInputs list then it will
55    not be use to create a metadata item.
56    """
57    inputs = context.getCopyOfInputs()
58   
59    metadataList = metadata_utils.constructMetadata(context, 
60                                                    excludedMetadataInputs,
61                                                    rt_type='list', 
62                                                    no_commas=False, 
63                                                    truncate=True,
64                                                    **inputs)
65    return metadataList
66
67   
68def processOutputs(context, fileSet, dryRun=False, keep=False):
69    """
70    Decide whether to zip the contents of I{context.outputDir} and create
71    a FileSet object representing the final output.
72   
73    """
74   
75    maxRequestVolume = float(context.config['max_request_volume'])
76    maxFileVolume = float(context.config['max_file_volume'])
77   
78#    log.debug("Fileset size before zipping = %s" % (sum([e.size for e in fileSet.contents]),) )
79#    log.debug("maxRequestVolume = %s" % (maxRequestVolume,))
80   
81    if dryRun:       
82        fileSet = dryRunZipFileSet(fileSet, context, maxFileVolume)
83       
84        # check that the fileset isn't too big
85        totalSize = sum([e.size for e in fileSet.contents])
86#        log.debug("Fileset size after zipping = %s" % (totalSize,))
87       
88        if totalSize > maxRequestVolume:
89           
90            context.log.error("totalSize = %s, maxRequestVolume = %s" , totalSize, maxRequestVolume )
91            raise Exception("Your request exceeds the maximum volume limit. Please try submitting smaller requests.")
92       
93    else:
94        fileSet = zipFileSet(fileSet, context, maxFileVolume, keep = keep)
95   
96    return fileSet
97
98def updateJobDetailsAfterCompletion(context, startTime, completionTime):
99    "Updates the job details to reflect state of the job after execution"
100   
101    _updateJobStatus(context)
102    _updateJobVolume(context)
103   
104    duration = completionTime - startTime
105    _setJobDuration(context, duration)
106   
107    _setJobCompletionTime(context, completionTime)
108   
109    _setDownloadCapability(context)
110   
111def updateJobDetailsAfterDryRun(context, duration):
112    "Updates the job details with the results of the dry run"
113   
114    _updateJobStatus(context)
115    _updateJobVolume(context)
116    _setJobDuration(context, duration)
117    _setDownloadCapability(context)
118           
119def _setDownloadCapability(context):
120   
121    jobDetails = context.outputs['job_details']
122    fileSet = context.outputs['FileSet']
123
124    if len(fileSet.contents) == 1:
125        if jobDetails.get('job_capabilities') in [None, ""]:
126            jobDetails['job_capabilities'] = 'download'
127        else:
128            jobDetails['job_capabilities'] += ' download'
129           
130def _updateJobStatus(context):
131    "Updates the jobDetails 'status' entry using the context object"
132   
133    jobDetails = context.outputs['job_details']
134    jobDetails['status'] = [context.status, context.statusMessage, context.percentComplete]
135   
136def _updateJobVolume(context):
137    "Updates the JobVolume by examining the current fileset in context.outputs"
138   
139    jobDetails = context.outputs['job_details']
140    fileSet = context.outputs['FileSet']
141   
142    total_size = 0
143    for file in fileSet.contents:
144        total_size += file.size
145       
146    jobDetails["job_volume"] = total_size
147
148def _setJobDuration(context, duration):
149    "Sets the 'duration' value in the jobDetails dictionary to the value given"
150   
151    jobDetails = context.outputs['job_details']
152    jobDetails["duration"] = duration
153
154def _setJobCompletionTime(context, completionTime):
155    """
156    Sets the jobDetails 'completion_time' and 'completion_date_time' based on
157    the timestamp given.
158    """
159    jobDetails = context.outputs['job_details']
160    jobDetails['completion_time'] = completionTime
161   
162    completionDateTime = datetime.fromtimestamp(completionTime)
163    jobDetails['completion_date_time'] = completionDateTime.isoformat(' ')
164   
Note: See TracBrowser for help on using the repository browser.