Changeset 6873


Ignore:
Timestamp:
20/05/10 10:45:21 (9 years ago)
Author:
astephen
Message:

Added and updated various code to match security, download URLs that are
secured and chunking of MIDAS outputs.

Location:
cows_wps/trunk
Files:
6 added
7 edited

Legend:

Unmodified
Added
Removed
  • cows_wps/trunk/cows_wps/controllers/download.py

    r5615 r6873  
    2626class DownloadController(BaseController): 
    2727 
    28     def index(self, jobId, fileName): 
     28    def index(self, jobId, authorisedRoles, fileName): 
    2929 
    3030        req = requestManager.getRequest(jobId, sgeUpdate=False) 
  • cows_wps/trunk/cows_wps/controllers/wps.py

    r6116 r6873  
    135135         
    136136    def Execute(self): 
     137        if request.environ.get("REQUEST_METHOD") == "POST": 
     138            log.warn("POST request was rejected because we do not allow this yet.") 
     139            abort(405) 
     140 
    137141        log.info("entering controllers.wps.Execute") 
    138142 
  • cows_wps/trunk/cows_wps/templates/defs.xml

    r5615 r6873  
    4848    <py:def function="markupFileSetContents(fileSet, jobId)"> 
    4949        <FileDetails py:for="file in fileSet.contents"> 
    50             <FileURL py:content="url_for(controller='/download', jobId=jobId, fileName=file.name, qualified=True)"/> 
     50            <FileURL py:content="url_for(controller='/download', authorisedRoles='1', jobId=jobId, fileName=file.name, qualified=True)"/> 
    5151            <FileSize py:content="file.size"/> 
    5252            <FileInfo py:content="file.info"/> 
  • cows_wps/trunk/cows_wps/utils/duration_splitter.py

    r6867 r6873  
    101101            ct = self._addDay(ct) 
    102102 
    103             if (self._isLastDayOfMonth(ct) and chunk_size == "month") or \ 
    104                    (self._isLastDayOfYear(ct) and chunk_size == "year") or \ 
    105                    (self._isLastDayOfDecade(ct) and chunk_size == "decade"): 
     103            if (self._isLastDayOfMonth(ct) and chunk_unit == "month") or \ 
     104                   (self._isLastDayOfYear(ct) and chunk_unit == "year") or \ 
     105                   (self._isLastDayOfDecade(ct) and chunk_unit == "decade"): 
    106106 
    107107                this_chunk.append(ct) 
  • cows_wps/trunk/process_configs/ExtractUKStationData.ini

    r6125 r6873  
    2626EndDateTime = datetime 
    2727EndDateTime.default = now 
     28OutputTimeChunk = string 
     29OutputTimeChunk.default = year 
     30OutputTimeChunk.possible_values = decade,year,month 
    2831BBox = bbox 
    2932BBox.optional = True 
  • cows_wps/trunk/process_modules/extract_uk_station_data.py

    r6126 r6873  
    1313from cows_wps.process_handler.context.process_status import STATUS 
    1414 
     15import process_modules.lib_get_weather_stations.utils as gws_utils 
     16import process_modules.lib_extract_uk_station_data.utils as exuk_utils 
     17 
    1518# Import local modules 
    1619sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract") 
    1720# MIDAS Station search code 
    18 import getStations 
     21#import getStations 
    1922import midasSubsetter 
    2023 
     
    3841        # Get required start/end times 
    3942        StartDateTime = ci['StartDateTime'] 
    40         StartDateTime = self._revertDateTimeToLongString(StartDateTime) 
     43        StartDateTime = gws_utils.revertDateTimeToLongString(StartDateTime) 
    4144        EndDateTime = ci['EndDateTime'] 
    42         EndDateTime = self._revertDateTimeToLongString(EndDateTime) 
     45        EndDateTime = gws_utils.revertDateTimeToLongString(EndDateTime) 
     46        TimeChunk = ci['OutputTimeChunk'] 
    4347 
    4448        ObsTableName = ci['ObsTableName']  
     
    5862                raise Exception("Invalid arguments provided. Must provide either a list of station IDs, a list of counties or a valid geographical bounding box.") 
    5963 
     64            # Call code to get Weather Stations 
     65            StationIDs = gws_utils.getStationList(Counties, BBox, DataTypes, StartDateTime, 
     66                           EndDateTime, sf_path) 
     67 
    6068            # Call code to get Weather Stations and write file 
    61             st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes, 
    62                        startTime=StartDateTime, endTime=EndDateTime, outputFile=sf_path, noprint=1) 
    63             StationIDs = st_getter.stList 
     69#            st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes, 
     70#                       startTime=StartDateTime, endTime=EndDateTime, outputFile=sf_path, noprint=1) 
     71#            StationIDs = st_getter.stList 
    6472            StationIDs.sort() 
    6573 
     
    7684 
    7785        # Pretend that took 10% of time 
    78         context.setStatus(STATUS.COMPLETED, 'The End', 10) 
     86        context.setStatus(STATUS.STARTED, 'Extracted station ID list.', 10) 
    7987 
    8088        # Now extract the data itself 
    81         data_file_root = "station_data" 
     89        data_file_prefix = "station_data" 
    8290        if Delimiter == "comma": 
    83             data_file = data_file_root + ".csv" 
     91            ext = "csv" 
    8492        else: 
    85             data_file = data_file_root + ".txt" 
     93            ext = "txt" 
    8694 
    87         df_path = context.processDir + '/outputs/' + data_file 
     95        df_path_base = context.processDir + '/outputs/' + data_file_prefix 
    8896 
    8997        # Need temp dir for big file extractions 
    9098        process_tmp_dir = context.processDir + '/tmp' 
    9199 
    92         startTime12Chars = StartDateTime[:-2] 
    93         endTime12Chars = EndDateTime[:-2] 
    94   
    95         midasSubsetter.MIDASSubsetter([ObsTableName], df_path, startTime=startTime12Chars,  
    96                        endTime=endTime12Chars, src_ids=StationIDs,  
    97                        tempDir=process_tmp_dir) 
    98          
    99         self._addFileToFileSet(df_path, "Station data file.", FLAG.DATA) 
     100        output_file_paths = exuk_utils.extractStationDataByTimeChunk([ObsTableName], StartDateTime, EndDateTime, 
     101                       StationIDs, TimeChunk, df_path_base, ext, process_tmp_dir) 
     102#        startTime12Chars = StartDateTime[:-2] 
     103#        endTime12Chars = EndDateTime[:-2] 
     104 
     105#        midasSubsetter.MIDASSubsetter([ObsTableName], df_path, startTime=startTime12Chars,  
     106#                       endTime=endTime12Chars, src_ids=StationIDs,  
     107#                       tempDir=process_tmp_dir) 
     108        
     109        for df_path in output_file_paths:  
     110            self._addFileToFileSet(df_path, "Station data file.", FLAG.DATA) 
    100111 
    101112        context.setStatus(STATUS.COMPLETED, 'The End', 100) 
     
    111122 
    112123 
    113     def _revertDateTimeToLongString(self, dt): 
    114         """ 
    115         Turns a date/time into a long string as needed by midas code. 
    116         """ 
    117         return str(dt).replace("-", "").replace(" ", "").replace("T", "").replace(":", "") 
    118  
    119124    def dryRun(self, context): 
    120125 
  • cows_wps/trunk/process_modules/get_weather_stations.py

    r6124 r6873  
    1515from cows_wps.process_handler.context.process_status import STATUS 
    1616 
     17#sys.path.append("/usr/local/cwps/cows_wps") 
     18import process_modules.lib_get_weather_stations.utils as gws_utils 
     19 
    1720# Import local modules 
    18 sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract") 
     21#sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract") 
    1922# MIDAS Station search code 
    20 import getStations 
     23#import getStations 
    2124 
    2225log = logging.getLogger(__name__) 
     
    4043        DataTypes = context.inputs.get("DataTypes", []) 
    4144        StartDateTime = context.inputs['StartDateTime'] 
    42         StartDateTime = self._revertDateTimeToLongString(StartDateTime) 
     45        StartDateTime = gws_utils.revertDateTimeToLongString(StartDateTime) 
    4346        log.warn("DATE TIME: %s, type = %s" % (StartDateTime, type(StartDateTime))) 
    4447        EndDateTime = context.inputs['EndDateTime'] 
    45         EndDateTime = self._revertDateTimeToLongString(EndDateTime) 
     48        EndDateTime = gws_utils.revertDateTimeToLongString(EndDateTime) 
    4649         
    4750        context.setStatus(STATUS.STARTED, 'Job is now running', 0) 
     
    5558 
    5659        # Call code to get Weather Stations 
    57         st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes,  
    58                        startTime=StartDateTime, endTime=EndDateTime, outputFile=MyFilePath, noprint=1)  
     60        station_list = gws_utils.getStationList(Counties, BBox, DataTypes, StartDateTime, 
     61                           EndDateTime, MyFilePath) 
     62#        st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes,  
     63#                       startTime=StartDateTime, endTime=EndDateTime, outputFile=MyFilePath, noprint=1)  
    5964 
    6065        context.outputs['ProcessSpecificContent'] = { 
    61                                     "WeatherStations": " ".join(st_getter.stList)}  
     66                                    "WeatherStations": " ".join(station_list)}  
    6267 
    6368        context.log.info('Written output file: %s' % WFile) 
     
    7277        ddp_process_support.updateJobDetailsAfterCompletion(context, startTime, completionTime) 
    7378 
    74     def _revertDateTimeToLongString(self, dt): 
    75         """ 
    76         Turns a date/time into a long string as needed by midas code. 
    77         """ 
    78         return str(dt).replace("-", "").replace(" ", "").replace("T", "").replace(":", "")  
    7979 
    8080    def dryRun(self, context): 
Note: See TracChangeset for help on using the changeset viewer.