Changeset 4440


Ignore:
Timestamp:
18/11/08 15:57:58 (11 years ago)
Author:
cbyrom
Message:

Create new model, Deployment, to hold deployments info - for easy use
with templates. Delete the moles deployments code and implement code
with the new model + add containers in Atom - to allow deployments data
specified in links to be looked up.

Location:
exist/trunk/python/ndgUtils
Files:
1 added
4 edited

Legend:

Unmodified
Added
Removed
  • exist/trunk/python/ndgUtils/models/Atom.py

    r4427 r4440  
    2121from ndgUtils.models import MolesEntity as ME 
    2222import csml.parser as CsmlParser 
     23from ndgUtils.models import Deployment as Deployment 
    2324 
    2425 
     
    263264        self.summary = [] 
    264265        self.content = [] 
     266        # NB, this deployments data duplicates other atom data - and is only used for a  
     267        # convenient way to collect the info (by lookupDeploymentsInfo()) for use in templates 
     268        self.deployments = []     
    265269        self.csmlFile = None 
    266270        self.cdmlFile = None 
     
    424428        collectionPath += self.ME.providerID + "/" 
    425429        return collectionPath 
    426              
     430 
    427431 
    428432    def __addAtomTypeDataXML(self, root): 
     
    11181122 
    11191123         
    1120      
    1121 def addCSMLData(csml, aggregateCoverage=False): 
    1122     ''' 
    1123     Parse CSML data and add extracted info to the atom 
    1124     @param csml: csml file contents - or path to csml file 
    1125     @keyword aggregateCoverage: if set to True, only coverage data that extends the 
    1126     atom coverage data will be added 
    1127     ''' 
    1128     logging.info("Creating CSML data model") 
    1129     csmlDoc = CsmlParser.Dataset(file=self._csmlFile) 
    1130      
    1131     logging.info("Extracting info from CSML file") 
    1132     logging.debug("Got dataset ID: %s" %csmlDoc.id) 
    1133     self.setDatasetID(csmlDoc.id) 
    1134      
    1135     title = csmlDoc.name.CONTENT 
    1136     logging.debug("Got dataset name (title): '%s'" %title) 
    1137     # NB, if a title is specified (and not as the default value), it automatically is used in  
    1138     # place of anything in the granulite file 
    1139     if title and title != "NAME OF DATASET GOES HERE": 
    1140         logging.info("Title, '%s', extracted from CSML file" %title) 
    1141         if self.title: 
    1142             logging.info("- NB, this will override the title specified in the granulite file ('%s')" \ 
    1143                          %self.title) 
    1144         self.title = title 
    1145              
    1146     bbox1 = csmlDoc.getBoundingBox() 
    1147     bbox2 = csmlDoc.getCSMLBoundingBox() 
    1148     time = bbox2.getTimeLimits() 
    1149  
    1150     # now check for other parameters to add to granule 
    1151     # Firstly, extract the bounding envelope 
    1152     if bbox1: 
    1153         w, e = self.moveBox(bbox1[0],bbox1[2]) 
    1154         n, s = (bbox1[3], bbox1[1]) 
    1155  
    1156         if not aggregateCoverage or (not self.maxY or float(n) > float(self.maxY)): 
    1157             self.maxY = n 
    1158              
    1159         if not aggregateCoverage or (not self.minY or float(s) < float(self.minY)): 
    1160             self.minY = s 
    1161          
    1162         if not aggregateCoverage or (not self.minX or float(w) < float(self.minX)): 
    1163             self.minX = w 
    1164  
    1165         if not aggregateCoverage or (not self.maxX or float(e) > float(self.maxX)): 
    1166             self.maxX = e 
    1167          
    1168         logging.debug("Got bounding box data from file: (%s, %s) , (%s, %s)" \ 
    1169                       %(w, s, e, n)) 
    1170          
    1171         logging.debug("Updated atom bounding box data: (%s, %s) , (%s, %s)" \ 
    1172                       %(self.minX, self.minY, self.maxX, self.maxY)) 
    1173     else: 
    1174         logging.debug("No valid bounding box data found") 
    1175  
    1176     if time: 
    1177         t1 = formatDateYYYYMMDD(time[0]) 
    1178         if not aggregateCoverage or \ 
    1179             (not self.t1 or datetime.datetime.strptime(t1, YEAR_FORMAT) < \ 
    1180                 datetime.datetime.strptime(self.t1, YEAR_FORMAT)): 
    1181             self.t1 = t1 
    1182  
    1183         t2 = time[1] 
    1184         if t2 and t2 != 'None': 
    1185             t2 = formatDateYYYYMMDD(t2) 
     1124    def addCSMLData(csml, aggregateCoverage=False): 
     1125        ''' 
     1126        Parse CSML data and add extracted info to the atom 
     1127        @param csml: csml file contents - or path to csml file 
     1128        @keyword aggregateCoverage: if set to True, only coverage data that extends the 
     1129        atom coverage data will be added 
     1130        ''' 
     1131        logging.info("Creating CSML data model") 
     1132        csmlDoc = CsmlParser.Dataset(file=self._csmlFile) 
     1133         
     1134        logging.info("Extracting info from CSML file") 
     1135        logging.debug("Got dataset ID: %s" %csmlDoc.id) 
     1136        self.setDatasetID(csmlDoc.id) 
     1137         
     1138        title = csmlDoc.name.CONTENT 
     1139        logging.debug("Got dataset name (title): '%s'" %title) 
     1140        # NB, if a title is specified (and not as the default value), it automatically is used in  
     1141        # place of anything in the granulite file 
     1142        if title and title != "NAME OF DATASET GOES HERE": 
     1143            logging.info("Title, '%s', extracted from CSML file" %title) 
     1144            if self.title: 
     1145                logging.info("- NB, this will override the title specified in the granulite file ('%s')" \ 
     1146                             %self.title) 
     1147            self.title = title 
     1148                 
     1149        bbox1 = csmlDoc.getBoundingBox() 
     1150        bbox2 = csmlDoc.getCSMLBoundingBox() 
     1151        time = bbox2.getTimeLimits() 
     1152     
     1153        # now check for other parameters to add to granule 
     1154        # Firstly, extract the bounding envelope 
     1155        if bbox1: 
     1156            w, e = self.moveBox(bbox1[0],bbox1[2]) 
     1157            n, s = (bbox1[3], bbox1[1]) 
     1158     
     1159            if not aggregateCoverage or (not self.maxY or float(n) > float(self.maxY)): 
     1160                self.maxY = n 
     1161                 
     1162            if not aggregateCoverage or (not self.minY or float(s) < float(self.minY)): 
     1163                self.minY = s 
     1164             
     1165            if not aggregateCoverage or (not self.minX or float(w) < float(self.minX)): 
     1166                self.minX = w 
     1167     
     1168            if not aggregateCoverage or (not self.maxX or float(e) > float(self.maxX)): 
     1169                self.maxX = e 
     1170             
     1171            logging.debug("Got bounding box data from file: (%s, %s) , (%s, %s)" \ 
     1172                          %(w, s, e, n)) 
     1173             
     1174            logging.debug("Updated atom bounding box data: (%s, %s) , (%s, %s)" \ 
     1175                          %(self.minX, self.minY, self.maxX, self.maxY)) 
     1176        else: 
     1177            logging.debug("No valid bounding box data found") 
     1178     
     1179        if time: 
     1180            t1 = formatDateYYYYMMDD(time[0]) 
    11861181            if not aggregateCoverage or \ 
    1187                 (not self.t2 or datetime.datetime.strptime(t2, YEAR_FORMAT) > \ 
    1188                     datetime.datetime.strptime(self.t2, YEAR_FORMAT)): 
    1189                 self.t2 = t2 
    1190          
    1191         logging.debug("Got time range: %s -> %s" %(self.t1, self.t2)) 
    1192     else: 
    1193         logging.debug("No valid time range data found") 
    1194  
    1195     #create parameter summaries: 
    1196     #set up list to hold the parameters data 
    1197     parameters = [] 
    1198     for feature in csmlDoc.featureCollection.featureMembers: 
    1199         if hasattr(feature.parameter, 'href'): 
    1200             paramTriple = "" 
    1201             if hasattr(feature, 'description'): 
    1202                 paramTriple = feature.description.CONTENT 
    1203                 paramTriple += " | " + feature.parameter.href 
    1204                  
    1205                 term = "" 
    1206                 if hasattr(feature, 'name'): 
    1207                     term = feature.name.CONTENT 
    1208  
    1209                 paramTriple += " | " + term 
    1210                  
    1211                 logging.debug("Got parameter info: %s" %paramTriple) 
    1212                 parameters.append(paramTriple) 
    1213      
    1214     # update the atom with the extracted parameters 
    1215     logging.info("Adding CSML parameters to granule atom") 
    1216     self.addParameters(parameters) 
    1217     logging.info("Finished adding CSML data") 
     1182                (not self.t1 or datetime.datetime.strptime(t1, YEAR_FORMAT) < \ 
     1183                    datetime.datetime.strptime(self.t1, YEAR_FORMAT)): 
     1184                self.t1 = t1 
     1185     
     1186            t2 = time[1] 
     1187            if t2 and t2 != 'None': 
     1188                t2 = formatDateYYYYMMDD(t2) 
     1189                if not aggregateCoverage or \ 
     1190                    (not self.t2 or datetime.datetime.strptime(t2, YEAR_FORMAT) > \ 
     1191                        datetime.datetime.strptime(self.t2, YEAR_FORMAT)): 
     1192                    self.t2 = t2 
     1193             
     1194            logging.debug("Got time range: %s -> %s" %(self.t1, self.t2)) 
     1195        else: 
     1196            logging.debug("No valid time range data found") 
     1197     
     1198        #create parameter summaries: 
     1199        #set up list to hold the parameters data 
     1200        parameters = [] 
     1201        for feature in csmlDoc.featureCollection.featureMembers: 
     1202            if hasattr(feature.parameter, 'href'): 
     1203                paramTriple = "" 
     1204                if hasattr(feature, 'description'): 
     1205                    paramTriple = feature.description.CONTENT 
     1206                    paramTriple += " | " + feature.parameter.href 
     1207                     
     1208                    term = "" 
     1209                    if hasattr(feature, 'name'): 
     1210                        term = feature.name.CONTENT 
     1211     
     1212                    paramTriple += " | " + term 
     1213                     
     1214                    logging.debug("Got parameter info: %s" %paramTriple) 
     1215                    parameters.append(paramTriple) 
     1216         
     1217        # update the atom with the extracted parameters 
     1218        logging.info("Adding CSML parameters to granule atom") 
     1219        self.addParameters(parameters) 
     1220        logging.info("Finished adding CSML data") 
     1221 
     1222 
     1223    def lookupDeploymentsInfo(self, dr): 
     1224        ''' 
     1225        Check through the atom links and retrieve any associated deployments 
     1226        data 
     1227        @param dr: Instance of DocumentRetrieve object - NB, this requires eXist 
     1228        config details which are not available to the Atom object 
     1229        ''' 
     1230        logging.info("Looking up deployments info") 
     1231        self.deployments = [] 
     1232        self.allActivities = [] 
     1233        self.allObs = [] 
     1234        self.allDpts = [] 
     1235        links = self.getLinksOfType(self.VTD.DEPLOYMENT_TERM) 
     1236 
     1237        for link in links: 
     1238            localID = link.href.split("__ATOM__")[-1] 
     1239            deploymentAtom = dr.get(self.ME.providerID, 'ATOM', localID, \ 
     1240                                    targetCollection='/db/atoms/') 
     1241            deployment = Deployment.Deployment(Atom(xmlString=str(deploymentAtom))) 
     1242            self.deployments.append(deployment) 
     1243            self.allActivities.extend(deployment.activities) 
     1244            self.allObs.extend(deployment.obs) 
     1245            self.allDpts.extend(deployment.dpts) 
     1246         
     1247        logging.info("Finished looking up deployments info") 
     1248         
  • exist/trunk/python/ndgUtils/models/MolesEntity.py

    r4427 r4440  
    1616from ndgUtils.vocabtermdata import VocabTermData as VTD 
    1717from utilities import getTripleData, escapeSpecialCharacters, tidyUpParameters 
    18  
    19 class DeploymentLink(object): 
    20      
    21     def __init__(self, url, vocabUrl, title=None): 
    22         ''' 
    23         Constructor - initialise the deployment link object 
    24         ''' 
    25         self.url = url 
    26         self.vocabUrl = vocabUrl 
    27         self.title = title 
    28          
    29     def toXML(self): 
    30         logging.info("Creating XML version of Deployment link element") 
    31         link = self.toET() 
    32         tree = ET.ElementTree(link) 
    33         logging.info("XML version of Deployment link element created") 
    34         return tree 
    35          
    36     def toET(self): 
    37         logging.info("Creating ElementTree version of Deployment link element") 
    38         link = ET.Element("link") 
    39         link.attrib["href"] = self.url 
    40         link.attrib["rel"] = self.vocabUrl 
    41         if self.title: 
    42             link.attrib["title"] = tidyUpParameters(self.title) 
    43  
    44         logging.info("ElementTree version of Deployment element link created") 
    45         return link 
    46  
    47     def __cmp__(self, link): 
    48         if not link: 
    49             return -1 
    50          
    51         if isinstance(link, DeploymentLink): 
    52             if link.url == self.url and \ 
    53                 link.vocabUrl == self.vocabUrl and \ 
    54                 link.title == self.title: 
    55                 return 0 
    56  
    57         return 1 
    58      
    59 class Deployments(object): 
    60      
    61     def __init__(self, id, deploymentLinks, startDate=None, endDate=None): 
    62         ''' 
    63         Constructor - initialise the deployments object 
    64         ''' 
    65         self.id = id 
    66         # ensure deployments is an array 
    67         self.deploymentLinks = [] 
    68         if deploymentLinks: 
    69             self.addDeploymentLinks(deploymentLinks) 
    70         self.startDate = startDate 
    71         self.endDate = endDate 
    72          
    73     def toXML(self): 
    74         logging.info("Creating XML version of Deployments element") 
    75         deployments = self.toET() 
    76         tree = ET.ElementTree(deployments) 
    77         logging.info("XML version of Deployments element created") 
    78         return tree 
    79  
    80  
    81     def toET(self): 
    82         logging.info("Creating ElementTree version of Deployments element") 
    83         deploymentElement = ET.Element("moles:deployment") 
    84         deploymentElement.attrib["id"] = self.id 
    85         startElement = ET.SubElement(deploymentElement, "moles:startDate") 
    86         startElement.text = self.startDate 
    87         endElement = ET.SubElement(deploymentElement, "moles:endDate") 
    88         endElement.text = self.endDate 
    89          
    90         for deploymentLink in self.deploymentLinks: 
    91             deploymentElement.append(deploymentLink.toET()) 
    92         logging.info("ElementTree version of Deployments element created") 
    93         return deploymentElement 
    94      
    95      
    96     def addDeploymentLinks(self, deploymentLinks): 
    97         logging.info("Adding deployment links to deployment") 
    98         if type(deploymentLinks) != type([]): 
    99             deploymentLinks = [deploymentLinks] 
    100         for deploymentLink in deploymentLinks: 
    101             # do uniqueness check 
    102             if deploymentLink not in self.deploymentLinks: 
    103                 self.deploymentLinks.append(deploymentLink) 
    104         logging.info("Deployment links added to deployment") 
    10518         
    10619 
     
    14962        logging.debug("Finished adding responsible party data") 
    15063 
    151  
    152  
    153     def addDeployment(self, deploymentID, startDate=None, endDate=None): 
    154         ''' 
    155         Add a deployment to the dictionary - ensuring it is unique and has been formatted and tidied appropriately 
    156         @param deploymentID: ID of the deployment - acts as a key in the deployments dictionary 
    157         @keyword startDate: optional start date 
    158         @keyword endDate: optional end date 
    159         ''' 
    160         logging.info("Adding new deployment data to moles entity") 
    161         self.deployments[deploymentID] = Deployments(deploymentID, None, startDate=startDate, endDate=endDate) 
    162         logging.info("New deployment data added to moles entity") 
    163  
    164  
    165     def addDeploymentLink(self, deploymentID, contentURL, vocabURL, title=None): 
    166         ''' 
    167         Add a deployment link to the dictionary - ensuring it is unique and has been formatted and tidied appropriately 
    168         @param deploymentID: ID of the deployment - acts as a key in the deployments dictionary 
    169         @param contentURL: url to deployment content 
    170         @param vocabURL: url to vocab term identifying type of deployment content 
    171         @keyword title: title string to display with deployment link 
    172         ''' 
    173         logging.info("Adding new deployment link data to moles entity") 
    174         logging.debug("contentURL: '%s', vocabURL: '%s', title: '%s'" %(contentURL, vocabURL, title)) 
    175         deploymentLink = DeploymentLink(contentURL, vocabURL, title=title) 
    176  
    177         self._addDeploymentDetails(deploymentLink) 
    178  
    179         if deploymentID not in self.deployments: 
    180             logging.debug("Creating new deployment type: %s" %deploymentID) 
    181             self.deployments[deploymentID] = Deployments(deploymentID, deploymentLink) 
    182         else: 
    183             logging.debug("Adding to existing deployment") 
    184             deployment = self.deployments[deploymentID] 
    185             deployment.addDeploymentLinks(deploymentLink) 
    186  
    187         logging.info("New deployment link data added to moles entity") 
    188  
    189          
    190     def _addDeploymentDetails(self, deploymentLink): 
    191         ''' 
    192         Add deployment details to local arrays for easy access by templates 
    193         ''' 
    194         deploymentType = None 
    195         # first, determine type of deployment 
    196         if deploymentLink.vocabUrl.startswith(VTD.TERM_DATA[VTD.ACTIVITY_TERM].vocabURL): 
    197             deploymentType = self.activities 
    198         elif deploymentLink.vocabUrl.startswith(VTD.TERM_DATA[VTD.OBS_TERM].vocabURL): 
    199             deploymentType = self.obs 
    200         elif deploymentLink.vocabUrl.startswith(VTD.TERM_DATA[VTD.DPT_TERM].vocabURL): 
    201             deploymentType = self.dpts 
    202         else: 
    203             return 
    204  
    205         # Now check if this has already been added; return if so 
    206         for link in deploymentType: 
    207             if link == deploymentLink: 
    208                 return 
    209  
    210         deploymentType.append(deploymentLink) 
    211  
    21264         
    21365    def toXML(self): 
     
    24193            subElement = ET.SubElement(molesISO, "moles:providerID") 
    24294            subElement.text = self.providerID 
    243  
    244         if self.deployments: 
    245             deploymentsElement = ET.SubElement(molesISO, "moles:deployments") 
    246              
    247             for deployment in self.deployments: 
    248                 deploymentsElement.append(self.deployments[deployment].toET()) 
    24995             
    25096        # if there's a created date already defined, use this, otherwise assume this is the creation point 
     
    287133            self.createdDate = createdDate 
    288134             
    289         deploymentElements = tree.findall('molesISO/deployments/deployment') 
    290         for deploymentElement in deploymentElements: 
    291             self._addDeploymentData(deploymentElement) 
    292              
    293135        logging.info("Data ingested from tree") 
    294  
    295  
    296     def _addDeploymentData(self, deploymentElement): 
    297         ''' 
    298         Parse a deployment element and extract all the required data from it 
    299         @param deploymentElement: ElementTree element representing a deployment element 
    300         ''' 
    301         id = deploymentElement.attrib.get('id') 
    302         startDate = deploymentElement.findtext('startDate') 
    303         endDate = deploymentElement.findtext('endDate') 
    304         linkElements = deploymentElement.findall('link') 
    305         self.addDeployment(id, startDate, endDate) 
    306         for linkElement in linkElements: 
    307             contentURL = linkElement.attrib.get('href') 
    308             vocabURL = linkElement.attrib.get('rel') 
    309             title = linkElement.attrib.get('title') 
    310             self.addDeploymentLink(id, contentURL, vocabURL, title = title) 
    311              
     136            
    312137             
    313138    def fromString(self, xmlString): 
  • exist/trunk/python/ndgUtils/models/testmolesentity.py

    r4214 r4440  
    2828            <moles:responsibleParty><name>dom lowe</name><uri>www.badc.rl.ac.uk</uri><role>Metadata maintainer</role></moles:responsibleParty> \ 
    2929        </moles:responsibleParties> \ 
    30         <moles:deployments> \ 
    31             <moles:deployment id="' + DEPLOYMENT_ID + '"> \ 
    32             <moles:startDate>' + createdDate + '</moles:startDate> \ 
    33             <moles:endDate>' + createdDate + '</moles:endDate> \ 
    34             <link href="' + CONTENT_URL + '" rel="' + VOCAB_URL + '" title="' + DEPLOYMENT_LINK_TITLE + '"></link> \ 
    35          </moles:deployment> \ 
    36          </moles:deployments> \ 
    3730         <moles:datasetLanguage>English</moles:datasetLanguage> \ 
    3831         <moles:metadataLanguage>English</moles:metadataLanguage> \ 
     
    5851         
    5952         
    60     def testAddDeployment(self): 
    61         self.ME.addDeployment(self.DEPLOYMENT_ID, self.createdDate, self.createdDate) 
    62         if self.DEPLOYMENT_ID not in self.ME.deployments: 
    63             self.fail("Deployment not correctly added to moles entity") 
    64          
    65         deployment = self.ME.deployments[self.DEPLOYMENT_ID] 
    66         self.assertEquals(deployment.startDate, self.createdDate) 
    67         self.assertEquals(deployment.endDate, self.createdDate) 
    68          
    69          
    70     def testAddDeploymentLink(self): 
    71         self.ME.addDeploymentLink(self.DEPLOYMENT_ID, self.CONTENT_URL, self.VOCAB_URL, self.DEPLOYMENT_LINK_TITLE) 
    72         if self.DEPLOYMENT_ID not in self.ME.deployments: 
    73             self.fail("Deployment not correctly added to moles entity") 
    74          
    75         deployment = self.ME.deployments[self.DEPLOYMENT_ID] 
    76         link = deployment.deploymentLinks[0] 
    77         self.assertEquals(link.url, self.CONTENT_URL) 
    78         self.assertEquals(link.vocabUrl, self.VOCAB_URL) 
    79         self.assertEquals(link.title, self.DEPLOYMENT_LINK_TITLE) 
    80          
    81          
    8253    def testFromString(self): 
    8354        self.ME.fromString(self.xmlString) 
     
    9263        self.assertEquals(link.vocabUrl, self.VOCAB_URL) 
    9364        self.assertEquals(link.title, self.DEPLOYMENT_LINK_TITLE) 
    94          
    95  
    96     def testAddDeploymentDetails(self): 
    97         a = ME.DeploymentLink('http://blah', VTD.TERM_DATA[VTD.ACTIVITY_TERM].vocabURL, title='fred') 
    98         b = ME.DeploymentLink('http://blah', VTD.TERM_DATA[VTD.ACTIVITY_TERM].vocabURL, title='fred') 
    99         c = ME.DeploymentLink('http://blah', VTD.TERM_DATA[VTD.ACTIVITY_TERM].vocabURL, title='freddy') 
    100          
    101         self.ME._addDeploymentDetails(a) 
    102         self.assertEquals(self.ME.activities[0], a) 
    103          
    104         self.ME._addDeploymentDetails(b) 
    105         self.assertEquals(len(self.ME.activities), 1) 
    106          
    107         self.ME._addDeploymentDetails(c) 
    108         self.assertEquals(len(self.ME.activities), 2) 
    109          
    110         self.ME._addDeploymentDetails(c) 
    111         self.assertEquals(len(self.ME.activities), 2) 
    112          
    11365 
    11466 
  • exist/trunk/python/ndgUtils/vocabtermdata.py

    r4434 r4440  
    184184                 } 
    185185 
    186     ATOM_TYPES = [ACTIVITY_TERM, DE_TERM, DPT_TERM, GRANULE_TERM, OBS_TERM] 
     186    ATOM_TYPES = [ACTIVITY_TERM, DE_TERM, DEPLOYMENT_TERM, DPT_TERM, GRANULE_TERM, OBS_TERM] 
    187187     
    188188    DEPLOYABLE_ATOM_TYPES = [ACTIVITY_TERM, DPT_TERM, OBS_TERM] 
Note: See TracChangeset for help on using the changeset viewer.