Changeset 6124


Ignore:
Timestamp:
14/12/09 16:03:12 (10 years ago)
Author:
astephen
Message:

adding more good stuff for the midas extraction bits.

Location:
cows_wps/trunk
Files:
10 edited

Legend:

Unmodified
Added
Removed
  • cows_wps/trunk/cows_wps/controllers/submit.py

    r6116 r6124  
    1919        """ 
    2020        args = {} 
     21        textarea_ids = [] 
     22        if "_textarea_ids" in request.params.keys(): 
     23            textarea_ids = request.params.getone("_textarea_ids").strip().split() 
     24 
    2125        for key in request.params.keys(): 
    2226            # Ignore items prefixed with underscore 
     
    2731            if values[0] == "-- Please select --": 
    2832                del values[0] 
     33 
     34            if str(key) in textarea_ids: 
     35                value = request.params.getone(key) 
     36                values = self._handleTextAreaInput(value) 
    2937 
    3038            # Remove if no values provided 
     
    5058 
    5159        return redirect_to(wps_request)  
     60 
     61    def _handleTextAreaInput(self, value): 
     62        "Removes any newlines in value and returns as list of strings." 
     63        value = str(value).replace("\r", "").replace("\n", "").strip().split() 
     64        return value 
    5265 
    5366    def choose(self): 
  • cows_wps/trunk/cows_wps/lib/ui/proc_config.py

    r6117 r6124  
    138138                Click the 'Submit' button to submit your request.""" 
    139139 
    140         resp += """<form action="/submit" onSubmit="return validateInputs();"> 
     140        resp += """\n\n <form action="/submit" onSubmit="return validateInputs();"> 
    141141                <input type="hidden" name="proc_id" value="%s" /> 
     142                <input type="hidden" id="_textarea_ids" name="_textarea_ids" value="" /> 
    142143                <table border="0">\n""" % proc 
    143144 
     
    170171            if opt == True: 
    171172                instruction = "This input is optional." 
     173            if multiple == True: 
     174                instruction += " Multiple inputs are allowed." 
    172175 
    173176            # Now render them according to data type etc 
     
    192195                    instruction += " Please insert a value of type: %s." % tp 
    193196 
    194                 resp += fm.renderTextInput(name, dtype=tp, optional=opt, default=default) 
     197                resp += fm.renderTextInput(name, dtype=tp, optional=opt, default=default, multiple=multiple) 
    195198            
    196199            elif tp == "filepath": 
  • cows_wps/trunk/cows_wps/process_handler/validate_arguments.py

    r6117 r6124  
    1717import re 
    1818import dateutil.parser 
    19 import datetime 
    2019 
    2120import logging 
     
    145144        raise Exception("Item does not match date/time pattern.") 
    146145 
    147     date_time_value = dateutil.parser.parse("2008-02-23T12:12:12") 
     146    date_time_value = dateutil.parser.parse(item) 
    148147    return date_time_value 
    149148 
  • cows_wps/trunk/cows_wps/public/js/ui/submit_utils.js

    r6116 r6124  
    2121} 
    2222 
     23/*  
     24 function registerTextArea: 
     25  * register id of textarea to hidden variable called '_textarea_ids' 
     26    which server can then parse to decide how to handle them. 
     27*/ 
     28function registerTextArea(name) { 
     29        // Adds text area id to container hidden input 
     30        var tac = $("#_textarea_ids"); 
     31        var ta_ids = tac.val(); 
     32 
     33        // Add name 
     34        if (ta_ids == "") { 
     35            ta_ids = name; 
     36        } else { 
     37            ta_ids = ta_ids + " " + name; 
     38        } 
     39        tac.val(ta_ids); 
     40} 
     41 
  • cows_wps/trunk/cows_wps/public/js/ui/validate.js

    r6117 r6124  
    2121/*  
    2222function: validateTextInput(): 
    23    Validates a string input checking it conforms to a regext pattern. 
     23   Validates a string input checking it conforms to a regex pattern. 
    2424   Inputs: 
    2525     * input 
     
    3737} 
    3838 
     39/* 
     40function: validateTextAreaInput(): 
     41   Validates a string input checking it conforms to a regex pattern. 
     42   Inputs: 
     43     * input 
     44     * regex_match 
     45   Returns: 
     46     * true OR false: if valid or not. 
     47*/ 
     48function validateTextAreaInput(input, regex_match, msg) { 
     49    // need to split text area input by line and space 
     50    var items = input.replace("\n", "").split(/\s/);  
     51 
     52    for (var i = 0; i < items.length; i++) { 
     53        var item = items[i]; 
     54 
     55        if (item.match(regex_match) == null) { 
     56            alert("You have submitted an invalid input, please try again. " + msg) 
     57            return false; 
     58        } 
     59    } 
     60    return true; 
     61} 
     62 
     63 
    3964var validators = new Array(); 
    4065/* Validator is a pseudo-class */ 
     
    4974/* ValidatorMappings is a pseudo-class */ 
    5075function ValidatorMappings() { 
    51     this.ids = new Array("text"); 
    52     this.values = new Array(validateTextInput); 
     76    this.ids = new Array("text", "textarea"); 
     77    this.values = new Array(validateTextInput, validateTextAreaInput); 
    5378 
    5479    function getFunction(id) { 
     
    101126        // If optional then we can return true if empty 
    102127        var res = false; 
    103         if (optional == true) { 
    104             if (value == "") { 
    105                 res = true; 
    106             } 
     128        if ((optional == true) && (value == "")) { 
     129            res = true; 
    107130        } else { 
    108131            res = fnc.call(this, value, regex_match, msg); 
  • cows_wps/trunk/cows_wps/renderer/form_renderer.py

    r6117 r6124  
    3333        return html 
    3434 
    35     def renderTextInput(self, name, dtype, optional=False, default=None): 
     35    def renderTextInput(self, name, dtype, optional=False, default=None, multiple=False): 
    3636        """ 
    3737        Returns HTML for text input including onChange validator. 
     
    3939 
    4040        Populates with default value if not None. 
     41        If multiple is true then makes box bigger. 
    4142        """ 
    4243        if default == None: default = "" 
     
    5455 
    5556        (regex, msg) = validators[dtype] 
    56   
    57         html = """<input type="text" name="%s" id="%s" value="%s" onChange="validateTextInput(this.value, %s, '%s', %s);" /><br />\n""" % (name, name, default, regex, msg, js_opt) 
    58         html += """<script type="text/javascript">addValidator('%s', 'text', %s, '%s', %s);</script>\n""" % (name, regex, msg, js_opt) 
     57 
     58        if multiple == True: 
     59            on_change = "validateTextAreaInput";  
     60        else: 
     61            on_change = "validateTextInput"; 
     62 
     63        tag_contents = """name="%s" id="%s" value="%s" onChange="%s(this.value, %s, '%s', %s);" """ % (name, name, default, on_change, regex, msg, js_opt) 
     64 
     65        if multiple == True: 
     66            # Use textarea 
     67            html = """<textarea cols="50" rows="3" %s ></textarea>""" % tag_contents 
     68            html += """\n<script type="text/javascript">registerTextArea('%s');</script>""" % name 
     69 
     70            validator_type = "textarea" 
     71        else: 
     72            # Use text input 
     73            html = '<input type="text" %s />' % tag_contents 
     74            validator_type = "text" 
     75 
     76        html += """<br />\n<script type="text/javascript">addValidator('%s', '%s', %s, '%s', %s);</script>\n""" % (name, validator_type, regex, msg, js_opt) 
    5977        return html 
    6078 
  • cows_wps/trunk/process_configs/ExtractUKStationData.ini

    r6116 r6124  
    3333Counties.possible_values = ABERDEENSHIRE,ALDERNEY,ANGUS,ANTRIM,ARGYLL (IN HIGHLAND REGION),ARGYLL (IN STRATHCLYDE REGION),ARGYLLSHIRE,ARMAGH,ASCENSION IS,AUSTRALIA (ADDITIONAL ISLANDS),AVON,AYRSHIRE,BANFFSHIRE,BEDFORDSHIRE,BERKSHIRE,BERWICKSHIRE,BORDERS,BOUVET ISLAND,BRECKNOCKSHIRE,BRITISH VIRGIN ISLANDS,BUCKINGHAMSHIRE,BUTESHIRE,CAERNARFONSHIRE,CAITHNESS,CAMBRIDGESHIRE,CARDIGANSHIRE,CARLOW,CARMARTHENSHIRE,CAVAN,CAYMAN ISLANDS,CENTRAL,CHANNEL ISLANDS,CHESHIRE,CHRISTMAS ISLAND,CLACKMANNANSHIRE,CLARE,CLEVELAND,CLWYD,COCOS ISLAND,COOK ISLANDS,CORK,CORNWALL,CUMBERLAND,CUMBRIA,CYPRUS,DENBIGHSHIRE,DERBYSHIRE,DETACHED ISLANDS,DEVON,DONEGAL,DORSET,DOWN,DUBLIN,DUMFRIES & GALLOWAY,DUMFRIESSHIRE,DUNBARTONSHIRE,DURHAM,DYFED,EAST LOTHIAN,EAST SUSSEX,ESSEX,FALKLAND IS,FERMANAGH,FIFE,FLINTSHIRE,FORFARSHIRE,GALWAY,GLAMORGANSHIRE,GLOUCESTERSHIRE,GRAMPIAN,GREATER LONDON,GREATER MANCHESTER,GUADALOUPE,GUERNSEY,GWENT,GWYNEDD,HAMPSHIRE,HAWAII,HEREFORD,HEREFORD & WORCESTER,HERTFORDSHIRE,HIGHLAND,HUMBERSIDE,HUNTINGDONSHIRE,INVERNESS-SHIRE,ISLE OF ANGLESEY,ISLE OF MAN,ISLE OF WIGHT,ISLES OF SCILLY,JERSEY,KENT,KERRY,KILDARE,KILKENNY,KINCARDINESHIRE,KINROSS-SHIRE,KIRKCUDBRIGHTSHIRE,LANARKSHIRE,LANARKSHIRE,LANCASHIRE,LAOIS,LEICESTERSHIRE,LEITRIM,LIMERICK,LINCOLNSHIRE,LONDONDERRY,LONGFORD,LOTHIAN,LOUTH,MALDIVES,MALTA,MAYO,MEATH,MEDITERRANEAN ISLANDS,MERIONETHSHIRE,MERSEYSIDE,MIDDLESEX,MID GLAMORGAN,MIDLOTHIAN,MIDLOTHIAN (IN BORDERS REGION),MIDLOTHIAN (IN LOTHIAN REGION),MONAGHAN,MONMOUTHSHIRE,MONTGOMERYSHIRE,MORAY,MORAY (IN GRAMPIAN REGION),MORAY (IN HIGHLAND REGION),NAIRNSHIRE,NORFOLK,NORTHAMPTONSHIRE,NORTHUMBERLAND,NORTH YORKSHIRE,NOTTINGHAMSHIRE,OCEAN ISLANDS,OFFALY,ORKNEY,OXFORDSHIRE,PACIFIC ISLANDS NORTH OF EQUATOR,PEEBLESHIRE,PEMBROKESHIRE,PERTHSHIRE,PERTHSHIRE (IN CENTRAL REGION),PERTHSHIRE (IN TAYSIDE REGION),PHOENIX ISLANDS,POWYS,POWYS (NORTH),POWYS (SOUTH),RADNORSHIRE,RENFREWSHIRE,ROSCOMMON,ROSS & CROMARTY,ROXBURGHSHIRE,RUTLAND,SANTA CRUZ ISLANDS,SARK,SELKIRKSHIRE,SEYCHELLES,SHETLAND,SHROPSHIRE,SINGAPORE,SLIGO,SOLOMON ISLANDS,SOMERSET,SOUTHERN LINE ISLANDS,SOUTH GEORGIA,SOUTH GLAMORGAN,SOUTH ORKNEYS,SOUTH SHETLAND,SOUTH YORKSHIRE,SPAIN (CANARY ISLANDS),STAFFORDSHIRE,ST HELENA,STIRLING,STIRLING (IN CENTRAL REGION),STIRLING (IN STRATHCLYDE REGION),STRATHCLYDE,SUFFOLK,SURREY,SUSSEX,SUTHERLAND,TAYSIDE,TIPPERARY,TURKS & CAICOS ISLANDS,TYNE & WEAR,TYRONE,WARWICKSHIRE,WATERFORD,WESTERN ISLES,WEST GLAMORGAN,WEST LOTHIAN,WEST LOTHIAN (IN CENTRAL REGION),WEST LOTHIAN (IN LOTHIAN REGION),WESTMEATH,WEST MIDLANDS,WESTMORLAND,WEST SUFFOLK,WEST SUSSEX,WEST YORKSHIRE,WEXFORD,WICKLOW,WIGTOWNSHIRE,WILTSHIRE,WORCESTERSHIRE,YORKSHIRE 
    3434StationIDs = string.list 
     35StationIDs.length = 1-20000 
    3536StationIDs.optional = True 
    36 DataTypes = string.list 
    37 DataTypes.optional = True 
     37ObsTableName = string 
     38ObsTableName.possible_values = TMSL,RO,TD,WD,RD,RS,TH,WM,WH 
    3839Delimiter = string 
    3940Delimiter.possible_values = comma,tab 
  • cows_wps/trunk/process_modules/extract_uk_station_data.py

    r6116 r6124  
    77""" 
    88 
    9 import os, stat, time 
     9import os, stat, time, sys 
    1010 
    1111from cows_wps.process_handler.fileset import FileSet, FLAG 
     
    1616sys.path.append("/home/badc/software/datasets/ukmo-midas/scripts/extract") 
    1717# MIDAS Station search code 
     18import getStations 
    1819import midasSubsetter 
    19  
    20 # Set up logger 
    21 log = logging.getLogger(__name__) 
    22 log.setLevel(logging.DEBUG) 
    23  
    2420 
    2521 
    2622class ExtractUKStationData(object): 
     23 
    2724    def __call__(self, context): 
    2825         
     
    3431        Username = ci['Username'] 
    3532        StationIDs = ci.get("StationIDs", [])  
    36         Counties 
    37          
     33        Delimiter = ci["Delimiter"] 
     34        Counties = ci.get('Counties', []) 
     35        BBox = ci.get("BBox", None) 
     36        DataTypes = ci.get("DataTypes", []) 
     37 
     38        # Get required start/end times 
     39        StartDateTime = ci['StartDateTime'] 
     40        StartDateTime = self._revertDateTimeToLongString(StartDateTime) 
     41        EndDateTime = ci['EndDateTime'] 
     42        EndDateTime = self._revertDateTimeToLongString(EndDateTime) 
     43 
     44        ObsTableName = ci['ObsTableName']  
     45 
    3846        context.setStatus(STATUS.STARTED, 'Job is now running', 0) 
    3947 
    4048        # Always need a FileSet, even if empty 
    41         fileSet = context.outputs['FileSet'] = FileSet() 
     49        self.fileSet = context.outputs['FileSet'] = FileSet() 
    4250 
    43         context.outputs['ProcessSpecificContent'] = {}  
    44         MyFile = "uk_station_data.txt" 
    45         MyFilePath = context.processDir + '/outputs/' + MyFile 
     51        stations_file = "uk_stations.txt" 
     52        sf_path = context.processDir + '/outputs/' + stations_file 
    4653 
    4754        # Get station IDs if not provided 
    4855        if StationIDs == []: 
    49             # Try counties next 
    50             
     56            # Check we have either counties or bbox to search domain on 
     57            if Counties == [] and BBox == None: 
     58                raise Exception("Invalid arguments provided. Must provide either a list of station IDs, a list of counties or a valid geographical bounding box.") 
    5159 
    52         context.log.info('Written output file: %s' % MyFile) 
    53         filesize = os.stat(MyFilePath)[stat.ST_SIZE] 
     60            # Call code to get Weather Stations and write file 
     61            st_getter = getStations.StationIDGetter(Counties, bbox=BBox, dataTypes=DataTypes, 
     62                       startTime=StartDateTime, endTime=EndDateTime, outputFile=sf_path, noprint=1) 
     63            StationIDs = st_getter.stList 
     64            StationIDs.sort() 
    5465 
    55         fileSet = context.outputs['FileSet'] = FileSet() 
    56         fileSet.contents.append(FileSet(FLAG.DATA, MyFile, filesize, 'The only output')) 
     66        # else write the file one per station id per line 
     67        else: 
     68            StationIDs.sort() 
     69            fout = open(sf_path, "w") 
     70            fout.write("\r\n".join([str(st_id) for st_id in StationIDs]))  
     71            fout.close() 
     72 
     73        self._addFileToFileSet(sf_path, "Station IDs file used for extraction of station data.", FLAG.DATA) 
     74 
     75        context.outputs['ProcessSpecificContent'] = {"StationIDs": " ".join(StationIDs)} 
     76 
     77        # Pretend that took 10% of time 
     78        context.setStatus(STATUS.COMPLETED, 'The End', 10) 
     79 
     80        # Now extract the data itself 
     81        data_file_root = "station_data" 
     82        if Delimiter == "comma": 
     83            data_file = data_file_root + ".csv" 
     84        else: 
     85            data_file = data_file_root + ".txt" 
     86 
     87        df_path = context.processDir + '/outputs/' + data_file 
     88 
     89        # Need temp dir for big file extractions 
     90        process_tmp_dir = context.processDir + '/tmp' 
     91 
     92        midasSubsetter.MIDASSubsetter([ObsTableName], df_path, startTime=StartDateTime,  
     93                       endTime=EndDateTime, src_ids=StationIDs, tempDir=process_tmp_dir) 
     94 
     95        self._addFileToFileSet(df_path, "Station data file.", FLAG.DATA) 
    5796 
    5897        context.setStatus(STATUS.COMPLETED, 'The End', 100) 
    59          
    6098        completionTime = time.time() 
    6199        ddp_process_support.updateJobDetailsAfterCompletion(context, startTime, completionTime) 
    62100 
    63      
     101 
     102    def _addFileToFileSet(self, path, info, type=FLAG.DATA): 
     103        "Adds file to output file set." 
     104        f_size = os.path.getsize(path) 
     105        output_basename = os.path.basename(path) 
     106        self.fileSet.contents.append(FileSet(type, output_basename, f_size, info)) 
     107 
     108 
     109    def _revertDateTimeToLongString(self, dt): 
     110        """ 
     111        Turns a date/time into a long string as needed by midas code. 
     112        """ 
     113        return str(dt).replace("-", "").replace(" ", "").replace("T", "").replace(":", "") 
    64114 
    65115    def dryRun(self, context): 
     
    67117        # Not implemented for sync jobs 
    68118        pass 
     119 
  • cows_wps/trunk/process_modules/get_weather_stations.py

    r6116 r6124  
    3636            BBox = context.inputs.get("BBox", None) 
    3737            if BBox == None: 
    38                 raise Exception("Invalid arguments provided. Must provide either a bounding box or a list of counties.") 
     38                raise Exception("Invalid arguments provided. Must provide either a geographical bounding box or a list of counties.") 
    3939 
    40         DataTypes = [] 
     40        DataTypes = context.inputs.get("DataTypes", []) 
    4141        StartDateTime = context.inputs['StartDateTime'] 
    4242        StartDateTime = self._revertDateTimeToLongString(StartDateTime) 
  • cows_wps/trunk/process_modules/process_template.py

    r6116 r6124  
    88 
    99# Standard library imports 
    10 import os, stat, time 
     10import os, stat, time, sys 
    1111 
    1212# WPS imports 
     
    1717# Import process-specific modules 
    1818 
    19  
    20 # Set up logger 
    21 log = logging.getLogger(__name__) 
    22 log.setLevel(logging.DEBUG) 
     19# NOTE ABOUT LOGGING: 
     20# You can log with the context.log object 
    2321 
    2422 
Note: See TracChangeset for help on using the changeset viewer.