Changeset 4444 for exist/trunk


Ignore:
Timestamp:
19/11/08 11:55:14 (11 years ago)
Author:
cbyrom
Message:

Add new xquery to lookup deployment atoms associated with an atom ID +
add code to models to allow the use of this to retrieve this information
+ update tests + avoid doubly escaping special characters.

Location:
exist/trunk
Files:
1 added
5 edited

Legend:

Unmodified
Added
Removed
  • exist/trunk/python/ndgUtils/DocumentRetrieve.py

    r4165 r4444  
    2727class DocumentRetrieve(ndg_eXist): 
    2828    ''' This class provides a document retrieval service via the NDG exist interfaces ''' 
     29     
     30    ATOM_DEPLOYMENTS = 'ATOM-DEPLOYMENTS' 
    2931    def __init__(self, repository, pwfile='passwords.txt'): 
    3032        logging.info("Using repository, '%s'" %repository) 
     
    3436        self.repository=repository 
    3537        self.xq=ndgXqueries() 
    36         self.knownQueries={'DIF':'moles2dif','DC':'moles2DC','ISO19139':'moles2iso19139', 
    37                            'NDG-B0':'moles','NDG-B1':'molesObjectType','MDIP':'moles2mdip', 
    38                            'NDG-A0':'csml','NumSim':'numsim', 'ATOM':'atom', 'ATOM-BACKUP':'atom'} 
     38        self.knownQueries={'DIF':'moles2dif','DC':'moles2DC','ISO19139':'moles2iso19139', \ 
     39                           'NDG-B0':'moles','NDG-B1':'molesObjectType','MDIP':'moles2mdip', \ 
     40                           'NDG-A0':'csml','NumSim':'numsim', 'ATOM':'atom', \ 
     41                           'ATOM-BACKUP':'atom', self.ATOM_DEPLOYMENTS:'atomDeploymentsList'} 
    3942 
    4043    def _retrieveDoc(self, schema, xqtype, targetCollection, repository, localID): 
  • exist/trunk/python/ndgUtils/models/Atom.py

    r4440 r4444  
    200200        self.label = "" 
    201201 
    202     def fromString(self, linkString): 
    203         (self.label, self.scheme, self.term) = getTripleData(linkString) 
     202    def fromString(self, linkString, escapeSpecialCharacters=True): 
     203        ''' 
     204        Create Category from triple string of format, 'label | scheme | term' 
     205        @param linkString: triple string to create category with 
     206        @keyword escapeSpecialCharacters: if set to True, special characters in  
     207        triple string are escaped (default)  
     208        ''' 
     209        (self.label, self.scheme, self.term) = getTripleData(linkString, \ 
     210            doEscape=escapeSpecialCharacters) 
    204211         
    205212    def fromETElement(self, linkTag): 
     
    357364 
    358365 
    359  
    360366    def addUniqueRelatedLinks(self, links): 
    361367        ''' 
     
    363369        @param links: a Link or array of Links to add to the relatedLinks attribute  
    364370        ''' 
    365         logging.debug("Adding new related links") 
    366         if not links: 
    367             return 
    368          
    369         if type(links) is not list: 
    370             links = [links] 
    371          
    372         for link in links: 
    373             if type(link) is not Link: 
    374                 logging.warning("Link is not of 'Link' object type (type='%s') - skipping" %type(link)) 
    375                 continue 
    376             if link not in self.relatedLinks: 
    377                 logging.debug("- adding unique link") 
    378                 self.relatedLinks.append(link) 
    379         logging.debug("Finished adding links") 
     371        self.addUniqueLinks(self.relatedLinks, links) 
    380372         
    381373 
     
    538530            param = tidyUpParameters(param) 
    539531            category = Category() 
    540             category.fromString(param) 
     532            # NB, data already tidied up here, so set keyword to avoid this happening again 
     533            category.fromString(param, escapeSpecialCharacters=False) 
    541534 
    542535            # now check for uniqueness 
     
    699692        xmlString = xmlString.replace('gml:', '') 
    700693        xmlString = xmlString.replace('xmlns="http://www.w3.org/2005/Atom"', '') 
     694        xmlString = xmlString.replace('default:', '') 
    701695 
    702696        # now create elementtree with the XML string 
     
    11201114            return True 
    11211115        return False 
     1116     
     1117    def isDeployable(self): 
     1118        if (self.atomTypeID == VTD.ACTIVITY_TERM and self.subtypeID != VTD.DEPLOYMENT_TERM) or \ 
     1119            self.atomTypeID == VTD.DPT_TERM or \ 
     1120            self.atomTypeID == VTD.OBS_TERM: 
     1121            return True 
     1122        return False 
    11221123 
    11231124         
     
    12211222 
    12221223 
    1223     def lookupDeploymentsInfo(self, dr): 
     1224    def lookupDeploymentsInfo(self, dr, lookupIndirectReferences=False): 
    12241225        ''' 
    12251226        Check through the atom links and retrieve any associated deployments 
     
    12271228        @param dr: Instance of DocumentRetrieve object - NB, this requires eXist 
    12281229        config details which are not available to the Atom object 
     1230        @keyword lookupIndirectReferences: if True, the atom ID is used to search 
     1231        defined deployments to find those which reference it, otherwise only  
     1232        deployments data featured in the atom related links are processed  
    12291233        ''' 
    12301234        logging.info("Looking up deployments info") 
     
    12331237        self.allObs = [] 
    12341238        self.allDpts = [] 
    1235         links = self.getLinksOfType(self.VTD.DEPLOYMENT_TERM) 
     1239 
     1240        if lookupIndirectReferences: 
     1241            logging.info("Looking up references to this atom from other deployments") 
     1242            doc = dr.get(self.ME.providerID, dr.ATOM_DEPLOYMENTS, self.atomBrowseURL, \ 
     1243                                        targetCollection='/db/atoms/') 
     1244            # now need to turn this results set into actual atoms 
     1245            tree = ET.fromstring(doc) 
     1246            links = [] 
     1247            for atom in tree: 
     1248                logging.debug("- found reference in deployment") 
     1249                links.append(ET.tostring(atom)) 
     1250             
     1251        else: 
     1252            links = self.getLinksOfType(self.VTD.DEPLOYMENT_TERM) 
    12361253 
    12371254        for link in links: 
    1238             localID = link.href.split("__ATOM__")[-1] 
    1239             deploymentAtom = dr.get(self.ME.providerID, 'ATOM', localID, \ 
    1240                                     targetCollection='/db/atoms/') 
     1255            if lookupIndirectReferences: 
     1256                deploymentAtom = link 
     1257            else: 
     1258                localID = link.href.split("__ATOM__")[-1] 
     1259                deploymentAtom = dr.get(self.ME.providerID, 'ATOM', localID, \ 
     1260                                        targetCollection='/db/atoms/') 
     1261 
    12411262            deployment = Deployment.Deployment(Atom(xmlString=str(deploymentAtom))) 
    12421263            self.deployments.append(deployment) 
    1243             self.allActivities.extend(deployment.activities) 
    1244             self.allObs.extend(deployment.obs) 
    1245             self.allDpts.extend(deployment.dpts) 
     1264             
     1265            self.addUniqueLinks(self.allActivities, deployment.activities) 
     1266            self.addUniqueLinks(self.allObs, deployment.obs) 
     1267            self.addUniqueLinks(self.allDpts, deployment.dpts) 
    12461268         
    12471269        logging.info("Finished looking up deployments info") 
    1248          
     1270 
     1271 
     1272 
     1273    def addUniqueLinks(self, dataArray, links): 
     1274        ''' 
     1275        Add links to specified array - if they are not already included 
     1276        @param dataArray: a list, potentially arlready containing links 
     1277        @param links: a Link or array of Links to add to the dataArray  
     1278        ''' 
     1279        logging.debug("Adding new links") 
     1280        if not links: 
     1281            return 
     1282         
     1283        if type(links) is not list: 
     1284            links = [links] 
     1285         
     1286        for link in links: 
     1287            if type(link) is not Link: 
     1288                logging.warning("Link is not of 'Link' object type (type='%s') - skipping" %type(link)) 
     1289                continue 
     1290            if link not in dataArray: 
     1291                logging.debug("- adding unique link") 
     1292                dataArray.append(link) 
     1293        logging.debug("Finished adding links") 
  • exist/trunk/python/ndgUtils/models/MolesEntity.py

    r4440 r4444  
    1313        # if you've egged it this is the way it comes 
    1414        from ndgUtils.elementtree import cElementTree as ET 
    15 import sys, logging, re, datetime 
    16 from ndgUtils.vocabtermdata import VocabTermData as VTD 
    17 from utilities import getTripleData, escapeSpecialCharacters, tidyUpParameters 
     15import logging, datetime 
     16from utilities import escapeSpecialCharacters 
    1817         
    1918 
  • exist/trunk/python/ndgUtils/models/testatom.py

    r4427 r4444  
    9696        self.link.href = 'www.fred.com' 
    9797        self.link.rel = 'related' 
     98        self.link2 = Link() 
     99        self.link2.title = 'pete' 
     100        self.link2.href = 'www.pete.com' 
     101        self.link2.rel = 'self' 
    98102 
    99103         
     
    110114        self.assertEqual(self.atom.parameters[0].term, 'OZONE') 
    111115        self.assertEqual(self.atom.parameters[1].term, 'ERROR') 
    112         self.assertEqual(self.atom.parameters[2].term, 'HYDROGEN &') 
     116        self.assertEqual(self.atom.parameters[2].term, 'HYDROGEN &') 
    113117        self.assertEqual(self.atom.parameters[2].scheme, 'http://vocab.ndg.nerc.ac.uk/blah') 
    114118        self.assertEqual(self.atom.parameters[2].label.lower(), 'Hydrogen Peroxide < ppbv'.lower()) 
     
    154158        self.assertEqual(self.atom.cdmlFile, None) 
    155159        self.assertEqual(self.atom.title, self.title) 
    156         print self.atom.logos 
    157         self.assertEqual(self.atom.logos[0].href, self.logo) 
     160        self.assertEqual(self.atom.getLogos()[0].href, self.logo) 
    158161        self.assertEqual(self.atom.datasetID, self.datasetID) 
    159162        self.assertEqual(self.atom.minX, self.minX) 
     
    257260        self.assertFalse(self.link in self.atom.relatedLinks) 
    258261        self.assertFalse(link2 in self.atom.relatedLinks) 
    259          
     262 
     263    def testAddUniqueLinks(self): 
     264        data = [] 
     265        links = [self.link, self.link2, self.link] 
     266        self.atom.addUniqueLinks(data, links) 
     267        self.assertEqual(2, len(data)) 
     268 
     269    def testAddUniqueLinksWithInitialLinksDefined(self): 
     270        data = [self.link2] 
     271        links = [self.link, self.link2, self.link] 
     272        self.atom.addUniqueLinks(data, links) 
     273        self.assertEqual(2, len(data)) 
    260274         
    261275 
  • exist/trunk/python/ndgUtils/models/utilities.py

    r4427 r4444  
    1919esc_chars = {'\xb0':'°','°':'°'} 
    2020 
    21 def getTripleData(tripleString): 
     21def getTripleData(tripleString, doEscape=True): 
    2222    ''' 
    2323    Take a string as input and extract triple data into an array 
    2424    NB, if data not fully in triple form, return empty elements 
    2525    @param tripleString: string containing the triple data 
     26    @keyword doEscape: if True, escape special characters - e.g. '&' (default True)  
    2627    @return 1-D array with three elements, representing the data in the triple 
    2728    ''' 
    28     logging.debug("Splitting triple: %s" %tripleString) 
     29    logging.debug("Getting triple data: %s" %tripleString) 
     30    if doEscape: 
     31        tripleString = escapeSpecialCharacters(tripleString) 
     32         
    2933    data = tripleString.split('|') 
    3034    returnData = ["", "", ""] 
     
    3337        if i > 2: 
    3438            raise ValueError("Triple data has an extra '|' character in it (%s) - please fix and rerun" %tripleString) 
    35         val = val.strip() 
    36         returnData[i] = escapeSpecialCharacters(val) 
     39        returnData[i] = val.strip() 
    3740        i += 1 
    3841     
     42    logging.debug("- returning triple data in array") 
    3943    return returnData 
    4044 
     
    156160 
    157161    # avoid processing the url, if it has been set 
     162    # - NB, special characters are escaped by getTripleData 
    158163    data = getTripleData(params_string) 
    159164    newData = [] 
     
    175180            r += " | " + newData[1] 
    176181     
    177     # lastly, remove any special characters 
    178     r = escapeSpecialCharacters(r)  
    179182    logging.debug("Tidied parameter string is now: %s" %r) 
    180183    return r 
Note: See TracChangeset for help on using the changeset viewer.