source: cows_wps/trunk/cows_wps/tests/test_ddp_process_support.py @ 5615

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/cows_wps/trunk/cows_wps/tests/test_ddp_process_support.py@5615
Revision 5615, 11.8 KB checked in by spascoe, 11 years ago (diff)

COWS WPS package copied from
 http://proj.badc.rl.ac.uk/svn/dcip/cows-wps/trunk.

This is a stripped down version of the DDP WPS. Some features are
removed and others have been deactivated until we reimplement them in a
more generic way.

Line 
1import unittest
2import time
3import logging
4from datetime import datetime
5
6import nose.tools as nt
7
8from cows_wps.process_handler.fileset import FileSet
9import cows_wps.process_handler.ddp_process_support as ddp_process_support
10
11log = logging.getLogger(__name__)
12TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
13
14class TestJobDetailsUpdateAfterCompletion(unittest.TestCase):
15   
16    def setUp(self):
17        c = EmptyClass()
18
19        c.outputs = {'job_details':{},
20                     'FileSet':FileSet()}
21        self.context = c
22        self.context.log = logging.getLogger('context')
23        self.startTime = time.mktime(time.strptime('2008-01-01 10:22:00', TIME_FORMAT)) 
24        self.endTime = time.mktime(time.strptime('2008-01-01 12:44:30', TIME_FORMAT))
25
26        self.setupStatusAttributes()
27
28    def setupStatusAttributes(self):
29        self.context.status = 'STATUS_VALUE'
30        self.context.statusMessage = 'STATUS MESSAGE'
31        self.context.percentComplete = 50
32       
33    def setupFileSetWithMultipleRoots(self):
34       
35        root_1 = FileSet(name='root1', size=323)
36        root_2 = FileSet(name='root2', size=80)
37       
38        self.context.outputs['FileSet'].contents.extend([root_1, root_2])
39       
40    def setupFileSetWithSingleRoot(self):
41       
42        root_1 = FileSet(name='root1', size=403)
43       
44        self.context.outputs['FileSet'].contents.extend([root_1])
45       
46       
47
48    def tearDown(self):
49        pass
50
51    def test_001_StatusIsSet(self):
52        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
53                                        self.startTime, self.endTime)       
54       
55        expectedStatus = [self.context.status, self.context.statusMessage, self.context.percentComplete]
56        resultingStatus = self.context.outputs['job_details']['status'] 
57        assert  expectedStatus == resultingStatus
58             
59
60    def test_002_VolumeIsSet(self):
61        self.setupFileSetWithMultipleRoots()
62       
63        contentFiles = self.context.outputs['FileSet'].contents
64        expectedVolume = sum([x.size for x in contentFiles])       
65       
66        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
67                                        self.startTime, self.endTime)
68       
69        resultingVolume = self.context.outputs['job_details']['job_volume']
70       
71        assert expectedVolume == resultingVolume
72       
73    def test_003_DurationIsSetAfterCompletion(self):
74        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
75                                        self.startTime, self.endTime)       
76       
77        expectedDuration = self.endTime - self.startTime
78        resultingDuration = self.context.outputs['job_details']['duration']
79       
80        assert expectedDuration == resultingDuration
81       
82    def test_004_CompletionTimeIsSet(self):
83        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
84                                        self.startTime, self.endTime)       
85       
86        expectedCompationTime = datetime.fromtimestamp(self.endTime).isoformat(' ')
87        resultingCompetionTime = self.context.outputs['job_details']['completion_date_time']
88       
89        assert expectedCompationTime, resultingCompetionTime
90       
91    def test_005_DownloadCapabilityIsNotSetForMultipleRootFiles(self):
92        self.setupFileSetWithMultipleRoots()
93        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
94                                        self.startTime, self.endTime)       
95       
96        resultingJobCapability = self.context.outputs['job_details'].get('job_capabilities')
97       
98        assert resultingJobCapability == None
99       
100    def test_006_DownloadCapabilityIsSetForSingleRootFile(self):
101        self.setupFileSetWithSingleRoot()
102        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
103                                        self.startTime, self.endTime)       
104       
105        resultingJobCapability = self.context.outputs['job_details']['job_capabilities']
106        assert resultingJobCapability == "download"
107       
108    def test_007_DownloadCapabilityAppendsCorrectly(self):
109        self.setupFileSetWithSingleRoot()
110       
111        self.context.outputs['job_details']['job_capabilities'] = 'capability_before'
112       
113        ddp_process_support.updateJobDetailsAfterCompletion(self.context, 
114                                        self.startTime, self.endTime)       
115       
116        resultingJobCapability = self.context.outputs['job_details']['job_capabilities']
117       
118        assert resultingJobCapability == "capability_before download"
119       
120class TestJobDetailsUpdateAfterDryRun(unittest.TestCase):
121   
122    def setUp(self):
123        c = EmptyClass()
124
125        c.outputs = {'job_details':{},
126                     'FileSet':FileSet()}
127        self.context = c
128        self.context.log = logging.getLogger('context')
129        self.duration = 10.4
130
131        self.setupStatusAttributes()
132       
133    def setupStatusAttributes(self):
134        self.context.status = 'STATUS_VALUE'
135        self.context.statusMessage = 'STATUS MESSAGE'
136        self.context.percentComplete = 50
137       
138    def setupFileSetWithSingleRoot(self):
139       
140        root_1 = FileSet(name='root1', size=403)
141       
142        self.context.outputs['FileSet'].contents.extend([root_1])
143       
144    def setupFileSetWithMultipleRoots(self):
145       
146        root_1 = FileSet(name='root1', size=323)
147        root_2 = FileSet(name='root2', size=80)
148       
149        self.context.outputs['FileSet'].contents.extend([root_1, root_2])
150       
151    def tearDown(self):
152        pass
153
154    def test_001_StatusIsSet(self):
155        ddp_process_support.updateJobDetailsAfterDryRun(self.context, self.duration)       
156       
157        expectedStatus = [self.context.status, self.context.statusMessage, self.context.percentComplete]
158        resultingStatus = self.context.outputs['job_details']['status'] 
159        assert  expectedStatus == resultingStatus
160             
161
162    def test_002_VolumeIsSet(self):
163        self.setupFileSetWithSingleRoot()
164       
165        contentFiles = self.context.outputs['FileSet'].contents
166        expectedVolume = sum([x.size for x in contentFiles])       
167       
168        ddp_process_support.updateJobDetailsAfterDryRun(self.context, self.duration)
169       
170        resultingVolume = self.context.outputs['job_details']['job_volume']
171       
172        assert expectedVolume == resultingVolume
173       
174    def test_003_DurationIsSetAfterCompletion(self):
175        ddp_process_support.updateJobDetailsAfterDryRun(self.context, self.duration)       
176       
177        expectedDuration = self.duration
178        resultingDuration = self.context.outputs['job_details']['duration']
179       
180        assert expectedDuration == resultingDuration
181       
182    def test_004_DownloadCapabilityIsNotSetForMultipleRootFiles(self):
183        self.setupFileSetWithMultipleRoots()
184       
185        ddp_process_support.updateJobDetailsAfterDryRun(self.context, self.duration)   
186       
187        resultingJobCapability = self.context.outputs['job_details'].get('job_capabilities')
188       
189        assert resultingJobCapability == None
190       
191    def test_005_DownloadCapabilityIsSetForSingleRootFile(self):
192        self.setupFileSetWithSingleRoot()
193       
194        ddp_process_support.updateJobDetailsAfterDryRun(self.context, self.duration)       
195       
196        resultingJobCapability = self.context.outputs['job_details']['job_capabilities']
197        assert resultingJobCapability == "download"
198       
199    def test_006_DownloadCapabilityAppendsCorrectly(self):
200        self.setupFileSetWithSingleRoot()
201       
202        self.context.outputs['job_details']['job_capabilities'] = 'capability_before'
203       
204        ddp_process_support.updateJobDetailsAfterDryRun(self.context, self.duration)       
205       
206        resultingJobCapability = self.context.outputs['job_details']['job_capabilities']
207       
208        assert resultingJobCapability == "capability_before download"       
209
210class TestProcessOutput(unittest.TestCase):
211   
212    def setUp(self):
213       
214        self.oldDryZipFn = ddp_process_support.dryRunZipFileSet
215        self.oldZipFn = ddp_process_support.zipFileSet
216       
217        self.replaceDryRunZipFileSet()
218        self.replaceZipFileSet()
219       
220        self.fileSet = FileSet()
221        self.fileSet.contents.append(FileSet(name='root1', size=50))
222       
223        self.zippedFileSet = FileSet()
224        self.zippedFileSet.contents.append(FileSet(name='root1', size=20))
225       
226        self.context = EmptyClass()
227        self.context.config = {'max_request_volume':100,
228                               'max_file_volume':100}
229       
230        self.context.log = logging.getLogger('context')
231   
232    def tearDown(self):
233        ddp_process_support.dryRunZipFileSet = self.oldDryZipFn
234        ddp_process_support.zipFileSet = self.oldZipFn         
235
236    def replaceDryRunZipFileSet(self):
237       
238        self.newDryRunZipFileSetArgs = None
239       
240        def newDryRunZipFileSet(*args, **kwargs):
241            self.newDryRunZipFileSetArgs = args
242            return self.zippedFileSet
243           
244        ddp_process_support.dryRunZipFileSet = newDryRunZipFileSet
245   
246    def replaceZipFileSet(self):
247       
248        self.newZipFileSetArgs = None
249        self.newZipFileSetKwargs = None
250       
251        def newZipFileSet(*args, **kwargs):
252            self.newZipFileSetArgs = args
253            self.newZipFileSetKwargs = kwargs
254            return self.zippedFileSet
255           
256        ddp_process_support.zipFileSet = newZipFileSet
257   
258    def test_001_zipFileSetCalled(self):
259       
260        ddp_process_support.processOutputs(self.context, self.fileSet, dryRun=False, keep=True)
261       
262        assert self.newZipFileSetArgs[0] == self.fileSet
263        assert self.newZipFileSetArgs[1] == self.context
264        assert self.newZipFileSetArgs[2] == float(self.context.config['max_file_volume'])
265        assert self.newZipFileSetKwargs == {'keep':True}
266       
267    def test_002_dryRunZipFileSetCalled(self):
268       
269        ddp_process_support.processOutputs(self.context, self.fileSet, dryRun=True, keep=True)
270       
271        assert self.newDryRunZipFileSetArgs[0] == self.fileSet
272        assert self.newDryRunZipFileSetArgs[1] == self.context
273        assert self.newDryRunZipFileSetArgs[2] == float(self.context.config['max_file_volume'])
274       
275    def test_003_exceptionThrownWhenRequestTooBig(self):
276       
277        self.fileSet.contents[0].size = 100
278        self.context.config['max_request_volume'] = 99
279       
280        nt.assert_raises(Exception, ddp_process_support.processOutputs, 
281                         [self.context, self.fileSet])
282
283    def test_004_maxRequestVolumeIsEvaluatedAfterCompression(self):
284       
285        # In order to give the best estimate of the total size of the jobs
286        # outputs the max_request_volume should be the total size after
287        # compression, not before.
288       
289        self.fileSet.contents[0].size = 100
290        self.zippedFileSet.contents[0].size = 50
291        self.context.config['max_request_volume'] = 75
292       
293        ddp_process_support.processOutputs(self.context, self.fileSet, dryRun=True)
294       
295
296class TestWriteMetadataFile(unittest.TestCase):
297    pass
298    #!TODO: write this test
299       
300       
301class EmptyClass(object):
302    pass
303   
304def suite():
305    suite = unittest.TestSuite()
306    #use all methods that start with 'test' to make the suite
307    suite.addTest(unittest.makeSuite(TestJobDetailsUpdateAfterCompletion))
308    suite.addTest(unittest.makeSuite(TestJobDetailsUpdateAfterDryRun))
309    suite.addTest(unittest.makeSuite(TestProcessOutput))
310    suite.addTest(unittest.makeSuite(TestWriteMetadataFile))
311
312    return suite
313           
314if __name__ == '__main__':
315    from cows_wps.tests.test_log_setup import LOG_FORMAT_STRING
316    logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT_STRING)
317
318    unittest.TextTestRunner(verbosity=2).run(suite())
319   
Note: See TracBrowser for help on using the repository browser.