Changeset 5058 for ndgCommon


Ignore:
Timestamp:
03/03/09 15:14:28 (11 years ago)
Author:
cbyrom
Message:

Extend vocabserver client, providing a method to retrieve a vocab list
+ add new data models to handle the returned info + update interface
details and usage + add new file utility method for creating
directories + extend tests.

Location:
ndgCommon/trunk/ndg/common
Files:
1 added
7 edited
1 copied

Legend:

Unmodified
Added
Removed
  • ndgCommon/trunk/ndg/common/src/clients/http/vocabserverclient.py

    r4990 r5058  
    66import logging, urlparse 
    77from xml.etree import ElementTree as ET 
     8from generichttpclient import GenericHTTPClient 
     9from ndg.common.src.models.vocablist import VocabList 
     10from ndg.common.src.models.ndgObject import ndgObject 
    811from ndg.common.src.lib.utilities import httpify 
    9 from generichttpclient import GenericHTTPClient 
    10      
     12 
    1113class VocabServerClient(GenericHTTPClient): 
    1214    '''  
    1315    Provides a POX interface to the vocab server  
    1416    ''' 
    15     VOCAB_NS = '{urn:vocab/types}' 
    1617    def __init__(self,  
    1718                 path = "http://vocab.ndg.nerc.ac.uk/axis2/services/vocab/", 
     
    4849        doc = self.readURL(url) 
    4950        x=ET.fromstring(doc) 
    50         b=x.findall('*/%sbroadMatch' %self.VOCAB_NS) 
    51         n=x.findall('*/%snarrowMatch' %self.VOCAB_NS) 
    52         s=x.findall('*/%sexactMatch' %self.VOCAB_NS) 
    53         self.broader=[(i.findtext('%sentryTerm' %self.VOCAB_NS)) for i in b] 
    54         self.narrower=[(i.findtext('%sentryTerm' %self.VOCAB_NS)) for i in n] 
    55         self.synonyms=[(i.findtext('%sentryTerm' %self.VOCAB_NS)) for i in s] 
     51        b=x.findall('*/{%s}broadMatch' %ndgObject.VOCAB_NS) 
     52        n=x.findall('*/{%s}narrowMatch' %ndgObject.VOCAB_NS) 
     53        s=x.findall('*/{%s}exactMatch' %ndgObject.VOCAB_NS) 
     54        self.broader=[(i.findtext('{%s}entryTerm' %ndgObject.VOCAB_NS)) for i in b] 
     55        self.narrower=[(i.findtext('{%s}entryTerm' %ndgObject.VOCAB_NS)) for i in n] 
     56        self.synonyms=[(i.findtext('{%s}entryTerm' %ndgObject.VOCAB_NS)) for i in s] 
    5657        logging.debug("- returning info on subject term") 
    5758        return [self.broader,self.narrower,self.synonyms] 
    5859 
     60     
     61    def getList(self, listKey): 
     62        ''' 
     63        Retrieve contents of a list - or a specific term in a list 
     64        @param listKey: key to use to retrieve list - e.g. 
     65        for list: http://vocab.ndg.nerc.ac.uk/list/P071/11 
     66        for term: http://vocab.ndg.nerc.ac.uk/term/P071/11/CFV10N55 
     67        @raise SystemError if the service returns an error 
     68        @raise IOError if an invalid listKey is provided 
     69        @return VocabList with all data loaded - or None, if nothing retrieved 
     70        ''' 
     71        logging.debug("Retrieving list info for key, '%s'" %listKey) 
     72        url = '%sgetList?recordKey=%s'%(self.path, listKey) 
     73        doc = self.readURL(url) 
     74        et = ET.fromstring(doc) 
     75         
     76        list = VocabList(et = et) 
     77         
     78        # check for errors 
     79        if list.error != 'false': 
     80            raise SystemError("Problem occurred whilst looking up vocab data: error") 
     81         
     82        logging.debug("- returning info in Elementtree object") 
     83        return list 
  • ndgCommon/trunk/ndg/common/src/clients/reldb/interfacereldbclient.py

    r5007 r5058  
    99    def setupDBConnection(self, configFile): 
    1010        ''' 
    11         Get the default DB connection - by reading in data from the db config file 
     11        Set up and return default DB connection - by reading in data from the  
     12        db config file 
    1213        @param configFile: filename of config file to use 
    1314        @raise ValueError: if config file has incorrect data 
  • ndgCommon/trunk/ndg/common/src/clients/reldb/postgres/postgresclient.py

    r5007 r5058  
    2323        # set up the db connection, if a config file is specified 
    2424        if configFile: 
    25             self.setupDBConnection(configFile) 
     25            self.conn = self.setupDBConnection(configFile) 
    2626             
    2727        logging.info("- client instantiated") 
     
    3333        @param configFile: filename of config file to use 
    3434        @raise ValueError: if config file has incorrect data 
    35         @return: pgdb.connection object with connection to DB set up  
     35        @return: connection object with connection to DB set up  
    3636        ''' 
    3737        logging.info("Setting up connection to postgres DB") 
     
    4949        logging.info("Postgres DB connection now set up") 
    5050 
    51         self.conn = dbConnection 
     51        return dbConnection 
    5252     
    5353     
     
    8080        if not self.conn: 
    8181            # attempt to run with default password file 
    82             self.setupDBConnection(DEFAULT_CONFIG_FILE) 
     82            self.conn = self.setupDBConnection(DEFAULT_CONFIG_FILE) 
    8383             
    84             if not self.conn: 
    85                 logging.error("DB connection not yet set up - run setupDBConnection(configFile) before trying to run a SQL command") 
    86                 return None 
    87          
    8884        cursor = self.conn.cursor() 
    8985        try: 
  • ndgCommon/trunk/ndg/common/src/lib/fileutilities.py

    r4793 r5058  
    3434 
    3535    from holger@trillke.net 2002/03/18 
     36    @return normalised file path 
    3637    ''' 
    3738    logging.info("Creating dir: " + path)  
     
    3940    if not exists(dpath): 
    4041        makedirs(dpath) 
    41     status = normpath(abspath(path)) 
     42    npath = normpath(abspath(path)) 
    4243     
    43     if status == 0: 
    44         logging.info("Directory created successfully") 
    45     return status 
     44    logging.info("Directory created successfully") 
     45    return npath 
     46 
     47 
     48def createDir(dir): 
     49    ''' 
     50    Create a new dir, if required 
     51    @param dir: directory to create 
     52    @raise IOError: if problems occur whilst creating directory 
     53    ''' 
     54    logging.info("Creating directory, %s" %dir) 
     55    status = 0 
     56    if not os.path.exists(dir): 
     57        if not dir.endswith(os.sep): 
     58            dir += os.sep 
     59        makepath(dir) 
     60    else: 
     61        logging.info("- directory already exists") 
     62     
    4663 
    4764 
  • ndgCommon/trunk/ndg/common/src/models/ndgObject.py

    r5019 r5058  
    4040    CSML_NS = 'http://ndg.nerc.ac.uk/csml' 
    4141    XHTML_NS = "http://www.w3.org/1999/xhtml" 
     42    VOCAB_NS = 'urn:vocab/types' 
     43    SKOS_NS = 'http://www.w3.org/2004/02/skos/core#' 
    4244    CDML_DTD = 'http://www-pcmdi.llnl.gov/software/cdms/cdml.dtd' 
     45    RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" 
     46    RDFS_NS = "http://www.w3.org/2000/01/rdf-schema#" 
     47    DC_NS = "http://purl.org/dc/elements/1.1/" 
    4348         
    4449    # Group the doc types according to the source they should be retrieved from 
  • ndgCommon/trunk/ndg/common/src/models/vocablist.py

    r4810 r5058  
    11''' 
    2  Class representing moles data that is too structured to be included in the basic Atom elements 
     2 Class representing vocab list data - with access methods for the different records 
    33   
    4  @author: C Byrom, Tessella Jun 2008 
     4 @author: C Byrom, Tessella Feb 2009 
    55''' 
    66from xml.etree import cElementTree as ET 
    7 import logging, datetime, re 
     7import logging  
    88from ndg.common.src.lib.utilities import escapeSpecialCharacters 
    99from ndg.common.src.models.ndgObject import ndgObject 
     10from ndg.common.src.models.codetablerecord import CodeTableRecord 
     11from ndg.common.src.lib.utilities import httpify 
    1012 
    11 class MolesEntity(object): 
     13class VocabList(object): 
    1214     
    13     # URL which points to the browse service - NB ensure this is properly defined 
    14     BROWSE_URL = "http://ndg.badc.rl.ac.uk:8080/view" 
    15      
    16     def __init__(self): 
     15    def __init__(self, et = None): 
    1716        ''' 
    1817        Constructor - initialise the atom variables 
    1918        ''' 
    20         logging.info("Initialising MolesEntity") 
     19        logging.info("Initialising VocabList") 
    2120 
    22         # an array of Atom.Person objects 
    23         self.responsibleParties = [] 
    24         self.deployments = {} 
    25         self.abbreviation = None 
    26         self.providerID = None 
    27         self.createdDate = None 
    28                  
    29         self.datasetLanguage = 'English' 
    30         self.metadataLanguage = 'English' 
     21        # an array of CodeTableRecord objects 
     22        self.records = [] 
     23 
     24        # error messages associated with the list retrieval 
     25        self.error = None 
    3126         
    32         # helper arrays to produce summary info for atom in templates 
    33         self.activities = [] 
    34         self.obs = [] 
    35         self.dpts = [] 
    36         logging.info("MolesEntity initialised")       
     27        self.url = None 
     28         
     29        self.name = None 
     30         
     31        # these are the users who can edit this list 
     32        self.userIDs = [] 
    3733 
    38  
    39     def addResponsibleParty(self, partyVals): 
    40         ''' 
    41         Add a responsible party - using the input string format,  
    42         ' name | uri | role ' 
    43         ''' 
    44         logging.debug("Adding responsible party data") 
    45         from ndg.common.src.models.Atom import Person 
    46         if type(partyVals) is not list: 
    47             partyVals = [partyVals] 
    48          
    49         for vals in partyVals: 
    50             rp = Person(personType = Person.RESPONSIBLE_PARTY_TYPE) 
    51             rp.fromString(vals) 
    52             self.responsibleParties.append(rp) 
    53         logging.debug("Finished adding responsible party data") 
    54  
    55          
    56     def toXML(self): 
    57         ''' 
    58         Convert the atom into XML representation and return this 
    59         @return: xml version of atom 
    60         ''' 
    61         logging.info("Creating formatted XML version of MolesEntity") 
    62         molesEntity = ET.Element("moles:entity") 
    63         molesEntity.attrib["xmlns:moles"] = ndgObject.MOLES_NS 
    64          
    65         molesISO = ET.SubElement(molesEntity, "moles:molesISO") 
    66         datasetLanguage = ET.SubElement(molesISO, "moles:datasetLanguage") 
    67         datasetLanguage.text = self.datasetLanguage 
    68         metadataLanguage = ET.SubElement(molesISO, "moles:metadataLanguage") 
    69         metadataLanguage.text = self.metadataLanguage 
    70  
    71         if self.responsibleParties: 
    72             rpElement = ET.SubElement(molesISO, "moles:responsibleParties") 
    73             logging.info("Adding responsible parties info") 
    74             for party in self.responsibleParties: 
    75                 rpElement.append(party.toXML()) 
    76  
    77         if self.abbreviation: 
    78             logging.info("Adding abbreviation element") 
    79             abbreviationElement = ET.SubElement(molesISO, "moles:abbreviation") 
    80             abbreviationElement.text = self.abbreviation 
    81  
    82         if self.providerID: 
    83             logging.info("Adding providerID element") 
    84             subElement = ET.SubElement(molesISO, "moles:providerID") 
    85             subElement.text = self.providerID 
    86              
    87         # if there's a created date already defined, use this, otherwise assume this is the creation point 
    88         # so use current time 
    89         if not self.createdDate: 
    90             self.createdDate = datetime.datetime.today().strftime("%Y-%m-%dT%H:%M:%SZ") 
    91  
    92         createdElement = ET.SubElement(molesISO, "moles:created") 
    93         createdElement.text = self.createdDate 
    94  
    95         logging.info("XML version of MolesEntity created") 
    96         return molesEntity 
     34        if et: 
     35            self.fromET(et) 
     36        logging.info("VocabList initialised")       
    9737 
    9838 
    9939    def fromET(self, tree): 
    10040        ''' 
    101         Initialise MolesEntity object using an elementtree 
    102         @param tree: ElementTree with moles entity data 
     41        Initialise VocabList data using an elementtree 
     42        @param tree: ElementTree with vocab list data 
    10343        ''' 
    10444        logging.info("Ingesting data from ElementTree object") 
    105         authorElements = tree.findall('{%s}molesISO/{%s}responsibleParties/{%s}responsibleParty' \ 
    106                                       %(ndgObject.MOLES_NS, ndgObject.MOLES_NS, ndgObject.MOLES_NS)) 
    107         from ndg.common.src.models.Atom import Person 
    108         for authorElement in authorElements: 
    109             logging.debug("Adding atom author data") 
    110             author = Person(personType = Person.RESPONSIBLE_PARTY_TYPE) 
    111             author.fromETElement(authorElement) 
    112             self.responsibleParties.append(author) 
    113                  
    114         self.abbreviation = tree.findtext('{%s}molesISO/{%s}abbreviation' \ 
    115                                           %(ndgObject.MOLES_NS, ndgObject.MOLES_NS)) 
     45        recordElements = tree.findall('{%s}codeTableRecord' %ndgObject.VOCAB_NS) 
     46         
     47        self.records = [] 
     48        for record in recordElements: 
     49            self.records.append(CodeTableRecord(et = record)) 
    11650 
    117         self.providerID = tree.findtext('{%s}molesISO/{%s}providerID' \ 
    118                                         %(ndgObject.MOLES_NS, ndgObject.MOLES_NS)) 
    119  
    120         self.datasetLanguage = tree.findtext('{%s}molesISO/{%s}datasetLanguage' \ 
    121                                              %(ndgObject.MOLES_NS, ndgObject.MOLES_NS)) 
    122         self.metadataLanguage = tree.findtext('{%s}molesISO/{%s}metadataLanguage' \ 
    123                                               %(ndgObject.MOLES_NS, ndgObject.MOLES_NS)) 
    124  
    125         createdDate = tree.findtext('{%s}molesISO/{%s}created' \ 
    126                                     %(ndgObject.MOLES_NS, ndgObject.MOLES_NS)) 
    127         if createdDate: 
    128             logging.debug("Adding created date") 
    129             self.createdDate = createdDate 
    130              
     51        self.error = tree.findtext('{%s}error' %ndgObject.VOCAB_NS) 
    13152        logging.info("Data ingested from tree") 
    132             
    133              
    134     def fromString(self, xmlString): 
    135         ''' 
    136         Initialise MolesEntity object using an xmlString 
    137         @param xmlString: representation of atom as an XML string  
    138         ''' 
    139         logging.info("Ingesting data from XML string") 
    140          
    141         # now create elementtree with the XML string 
    142         logging.debug("Create elementtree instance with XML string") 
    143         # NB, there is every chance that the namespace will not be available in  
    144         # the input xmlString - since it is likely to be part of the Atom string 
    145         # passed in - and this will have the namespaces defined in the parent tag 
    146         # - check and add the moles NS if it is missing 
    147         xmlString = self.__addMolesNS(xmlString) 
    148          
    149         tree = ET.fromstring(xmlString) 
    150         self.fromET(tree) 
    151         logging.info("Completed data ingest from XML string") 
    15253 
    15354 
    154     def __addMolesNS(self, xmlString): 
     55    def setData(self, url, name, userIDs): 
    15556        ''' 
    156         Check an XML string representation of the moles entity to see if it 
    157         contains the required moles namespace; if not, add it 
     57        Set data on the vocab list data model 
     58        @param url: the url corresponding to the vocab term list 
     59        @param name: the name of the list 
     60        @param userIDs: a list of userIDs - mapping to the users that can 
     61        edit entries in this list   
    15862        ''' 
    159         logging.debug("Checking if entity requires moles namespace adding") 
    160         if xmlString.find(ndgObject.MOLES_NS) > -1: 
    161             logging.debug("- namespace found - returning entity as is") 
    162             return 
    163  
    164         logging.debug("- namespace not found - adding this now") 
    165          
    166         # firstly get the local namespace prefix name 
    167         prefix = re.search('<(\w+):', xmlString).group(1) 
    168         elementName = prefix + ':entity ' 
    169          
    170         # now do a replace on this to add in the namespace 
    171         xmlString = xmlString.replace(elementName, elementName + "xmlns:%s= '%s' " \ 
    172                                       %(prefix, ndgObject.MOLES_NS)) 
    173          
    174         return xmlString 
    175  
    176          
    177     def setAttribute(self, attributeName, attributeValue): 
    178         ''' 
    179         Set the value of an atom attribute - and do some basic tidying up of the string content 
    180         - to escape any XML unfriendly characters 
    181         @param attributeName: name of the attribute whose value to set 
    182         @param attributeValue: value to set the attribute to   
    183         ''' 
    184         logging.debug("Setting attribute, %s, to %s" %(attributeName, attributeValue)) 
    185         origValue = attributeValue 
    186          
    187         # escape any special characters if a value has been specified 
    188         # NB, need to cope with both single values and arrays 
    189         if attributeValue: 
    190             if type(attributeValue) is list: 
    191                 newVals = [] 
    192                 for val in attributeValue: 
    193                     newVals.append(escapeSpecialCharacters(val)) 
    194                 attributeValue = newVals 
    195             else: 
    196                 attributeValue = escapeSpecialCharacters(attributeValue)  
    197         setattr(self, attributeName, attributeValue) 
     63        logging.debug("Setting data on VocabList:\nurl = %s \nname = %s \nuserIDs = %s" 
     64                      %(url, name, userIDs)) 
     65        self.url = httpify(url) 
     66        self.name = name 
     67        self.userIDs = userIDs 
     68        logging.debug("Finished setting data on VocabList") 
  • ndgCommon/trunk/ndg/common/unittests/clients/http/testvocabserverclient.py

    r4990 r5058  
    1717import ndg.common.unittests.testconstants as tc 
    1818from ndg.common.src.models.myconfig import myConfig 
     19from ndg.common.src.models.vocablist import VocabList 
    1920 
    20 class TestCase(unittest.TestCase):  
     21class TestVocabServerClient(unittest.TestCase):  
    2122 
    2223    def setUp(self): 
     
    2829 
    2930     
    30     def testVocabServerClient1(self): 
     31    def testGetRelated1(self): 
    3132        res = self.client.getRelated('rain') 
    3233        self.assertEquals(res[1][0], 'rain') 
    3334 
    3435     
    35     def testVocabServerClient2(self): 
     36    def testGetRelated2(self): 
    3637        res = self.client.getRelated('snow') 
    3738        self.assertEquals(res[1][0], 'snow') 
    3839 
    3940         
    40     def testInvalidVocabServer(self): 
     41    def testInvalidGetRelated(self): 
    4142        res = self.client.getRelated('rainxxxxx') 
    4243        self.assertEquals([], res[0]) 
     
    4445        self.assertEquals(3, len(res)) 
    4546         
     47    def testGetList(self): 
     48        list = self.client.getList(tc.STANDARD_NAMES_LIST) 
     49        self.assertTrue(isinstance(list, VocabList)) 
     50        self.assertEqual('false', list.error) 
     51        self.assertTrue(len(list.records) > 0) 
     52         
     53    def testInvalidGetList(self): 
     54        self.assertRaises(IOError, self.client.getList, tc.STANDARD_NAMES_LIST + 'blah') 
     55         
     56         
  • ndgCommon/trunk/ndg/common/unittests/testconstants.py

    r5019 r5058  
    208208              '/db/moles_support/granuleAtom/famous_control_month_granule.moles'] 
    209209 
     210 
     211STANDARD_NAMES_LIST = "http://vocab.ndg.nerc.ac.uk/list/P071/11" 
Note: See TracChangeset for help on using the changeset viewer.