source: cows_wps/trunk/process_modules/lib_extract_uk_station_data/utils.py @ 7082

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/cows_wps/trunk/process_modules/lib_extract_uk_station_data/utils.py@7082
Revision 7082, 2.4 KB checked in by astephen, 10 years ago (diff)

getting status monitoring working properly.

Line 
1"""
2utils.py
3========
4
5Util functions for process extract_uk_station_data that holds the ExtractUKStationData class.
6
7"""
8
9import os, stat, time, sys
10import urllib
11
12from cows_wps.process_handler.context.process_status import STATUS
13from cows_wps.utils.duration_splitter import *
14
15# Import local modules
16sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract")
17# MIDAS Station search code
18import midasSubsetter
19
20
21def extractStationDataByTimeChunk(obs_tables, startTime, endTime, src_ids, time_chunk, 
22                    output_file_base, delimiter, ext, tempDir, context):
23    """
24    Loops through time chunks extracting data to files in required time chunks.
25    We pass the context object through here so that we can report progress.
26    Returns a list of output file paths produced.
27    """ 
28    start_hr_min = startTime[8:12]
29    end_hr_min = endTime[8:12]
30
31    # Split the time chunks appropriately
32    ds = DurationSplitter()
33    time_splits = ds.splitDuration(startTime[:8], endTime[:8], time_chunk)
34    first_date = True
35
36    # Create list to return
37    output_file_paths = []
38
39    progress = 5 # % of way through
40    for (start_date, end_date) in time_splits: 
41
42        # Add appropriate hours and minutes to date strings to make 12 character times
43        if first_date == True:
44            start = "%s%s" % (start_date.date, start_hr_min)
45            first_date = False
46        else:
47            start = "%s0000" % (start_date.date)
48       
49        if end_date == time_splits[-1][-1]:
50            end = "%s%s" % (end_date.date, end_hr_min)
51        else:
52            end = "%s2359" % (end_date.date)
53
54        # Decide the output file path
55        output_file_path = "%s-%s-%s.%s" % (output_file_base, start, end, ext)
56        output_file_paths.append(output_file_path)
57 
58        # Call subsetter to extract and write the data   
59        midasSubsetter.MIDASSubsetter(obs_tables, output_file_path, startTime = start,
60                       endTime = end, src_ids = src_ids, delimiter = delimiter, tempDir = tempDir)
61
62        # Report on progress
63        progress += (int(list(time_splits).index((start_date, end_date)) / len(time_splits)) * 100 - 5)
64        context.setStatus(STATUS.STARTED, 'Job is now running', progress)
65       
66    return output_file_paths
67
68
69def extractStationsFromFile(fpath):
70    """
71    Reads file and extracts stations to a list.
72    """
73    f = open(fpath)
74    data = f.read() 
75    f.close()
76
77    station_ids = data.split() 
78    return station_ids
Note: See TracBrowser for help on using the repository browser.