Ignore:
Timestamp:
20/05/10 10:45:21 (10 years ago)
Author:
astephen
Message:

Added and updated various code to match security, download URLs that are
secured and chunking of MIDAS outputs.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • cows_wps/trunk/process_modules/extract_uk_station_data.py

    r6126 r6873  
    1313from cows_wps.process_handler.context.process_status import STATUS 
    1414 
     15import process_modules.lib_get_weather_stations.utils as gws_utils 
     16import process_modules.lib_extract_uk_station_data.utils as exuk_utils 
     17 
    1518# Import local modules 
    1619sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract") 
    1720# MIDAS Station search code 
    18 import getStations 
     21#import getStations 
    1922import midasSubsetter 
    2023 
     
    3841        # Get required start/end times 
    3942        StartDateTime = ci['StartDateTime'] 
    40         StartDateTime = self._revertDateTimeToLongString(StartDateTime) 
     43        StartDateTime = gws_utils.revertDateTimeToLongString(StartDateTime) 
    4144        EndDateTime = ci['EndDateTime'] 
    42         EndDateTime = self._revertDateTimeToLongString(EndDateTime) 
     45        EndDateTime = gws_utils.revertDateTimeToLongString(EndDateTime) 
     46        TimeChunk = ci['OutputTimeChunk'] 
    4347 
    4448        ObsTableName = ci['ObsTableName']  
     
    5862                raise Exception("Invalid arguments provided. Must provide either a list of station IDs, a list of counties or a valid geographical bounding box.") 
    5963 
     64            # Call code to get Weather Stations 
     65            StationIDs = gws_utils.getStationList(Counties, BBox, DataTypes, StartDateTime, 
     66                           EndDateTime, sf_path) 
     67 
    6068            # Call code to get Weather Stations and write file 
    61             st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes, 
    62                        startTime=StartDateTime, endTime=EndDateTime, outputFile=sf_path, noprint=1) 
    63             StationIDs = st_getter.stList 
     69#            st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes, 
     70#                       startTime=StartDateTime, endTime=EndDateTime, outputFile=sf_path, noprint=1) 
     71#            StationIDs = st_getter.stList 
    6472            StationIDs.sort() 
    6573 
     
    7684 
    7785        # Pretend that took 10% of time 
    78         context.setStatus(STATUS.COMPLETED, 'The End', 10) 
     86        context.setStatus(STATUS.STARTED, 'Extracted station ID list.', 10) 
    7987 
    8088        # Now extract the data itself 
    81         data_file_root = "station_data" 
     89        data_file_prefix = "station_data" 
    8290        if Delimiter == "comma": 
    83             data_file = data_file_root + ".csv" 
     91            ext = "csv" 
    8492        else: 
    85             data_file = data_file_root + ".txt" 
     93            ext = "txt" 
    8694 
    87         df_path = context.processDir + '/outputs/' + data_file 
     95        df_path_base = context.processDir + '/outputs/' + data_file_prefix 
    8896 
    8997        # Need temp dir for big file extractions 
    9098        process_tmp_dir = context.processDir + '/tmp' 
    9199 
    92         startTime12Chars = StartDateTime[:-2] 
    93         endTime12Chars = EndDateTime[:-2] 
    94   
    95         midasSubsetter.MIDASSubsetter([ObsTableName], df_path, startTime=startTime12Chars,  
    96                        endTime=endTime12Chars, src_ids=StationIDs,  
    97                        tempDir=process_tmp_dir) 
    98          
    99         self._addFileToFileSet(df_path, "Station data file.", FLAG.DATA) 
     100        output_file_paths = exuk_utils.extractStationDataByTimeChunk([ObsTableName], StartDateTime, EndDateTime, 
     101                       StationIDs, TimeChunk, df_path_base, ext, process_tmp_dir) 
     102#        startTime12Chars = StartDateTime[:-2] 
     103#        endTime12Chars = EndDateTime[:-2] 
     104 
     105#        midasSubsetter.MIDASSubsetter([ObsTableName], df_path, startTime=startTime12Chars,  
     106#                       endTime=endTime12Chars, src_ids=StationIDs,  
     107#                       tempDir=process_tmp_dir) 
     108        
     109        for df_path in output_file_paths:  
     110            self._addFileToFileSet(df_path, "Station data file.", FLAG.DATA) 
    100111 
    101112        context.setStatus(STATUS.COMPLETED, 'The End', 100) 
     
    111122 
    112123 
    113     def _revertDateTimeToLongString(self, dt): 
    114         """ 
    115         Turns a date/time into a long string as needed by midas code. 
    116         """ 
    117         return str(dt).replace("-", "").replace(" ", "").replace("T", "").replace(":", "") 
    118  
    119124    def dryRun(self, context): 
    120125 
Note: See TracChangeset for help on using the changeset viewer.