source: cows_wps/trunk/process_modules/lib_extract_uk_station_data/utils.py @ 7084

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/cows_wps/trunk/process_modules/lib_extract_uk_station_data/utils.py@7084
Revision 7084, 2.5 KB checked in by astephen, 10 years ago (diff)

fixes to status setting

Line 
1"""
2utils.py
3========
4
5Util functions for process extract_uk_station_data that holds the ExtractUKStationData class.
6
7"""
8
9import os, stat, time, sys
10import urllib
11
12from cows_wps.process_handler.context.process_status import STATUS
13from cows_wps.utils.duration_splitter import *
14
15# Import local modules
16sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract")
17# MIDAS Station search code
18import midasSubsetter
19
20
21def extractStationDataByTimeChunk(obs_tables, startTime, endTime, src_ids, time_chunk, 
22                    output_file_base, delimiter, ext, tempDir, context):
23    """
24    Loops through time chunks extracting data to files in required time chunks.
25    We pass the context object through here so that we can report progress.
26    Returns a list of output file paths produced.
27    """ 
28    start_hr_min = startTime[8:12]
29    end_hr_min = endTime[8:12]
30
31    # Split the time chunks appropriately
32    ds = DurationSplitter()
33    time_splits = ds.splitDuration(startTime[:8], endTime[:8], time_chunk)
34    first_date = True
35
36    # Create list to return
37    output_file_paths = []
38
39    progress = 1 # % of way through
40
41    for (count, (start_date, end_date)) in enumerate(time_splits): 
42
43        # Add appropriate hours and minutes to date strings to make 12 character times
44        if first_date == True:
45            start = "%s%s" % (start_date.date, start_hr_min)
46            first_date = False
47        else:
48            start = "%s0000" % (start_date.date)
49       
50        if end_date == time_splits[-1][-1]:
51            end = "%s%s" % (end_date.date, end_hr_min)
52        else:
53            end = "%s2359" % (end_date.date)
54
55        # Decide the output file path
56        output_file_path = "%s-%s-%s.%s" % (output_file_base, start, end, ext)
57        output_file_paths.append(output_file_path)
58 
59        # Call subsetter to extract and write the data   
60        midasSubsetter.MIDASSubsetter(obs_tables, output_file_path, startTime = start,
61                       endTime = end, src_ids = src_ids, delimiter = delimiter, tempDir = tempDir)
62
63        # Report on progress
64        progress += (int(float(count) / len(time_splits) * 100))
65        context.setStatus(STATUS.STARTED, 'Job is now running', progress)
66        context.saveStatus()
67       
68    return output_file_paths
69
70
71def extractStationsFromFile(fpath):
72    """
73    Reads file and extracts stations to a list.
74    """
75    f = open(fpath)
76    data = f.read() 
77    f.close()
78
79    station_ids = data.split() 
80    return station_ids
Note: See TracBrowser for help on using the repository browser.