Changeset 4490
- Timestamp:
- 26/11/08 10:03:15 (12 years ago)
- Files:
-
- 3 added
- 2 deleted
- 9 edited
Legend:
- Unmodified
- Added
- Removed
-
exist/trunk/python/ndgUtils/models/AccessControl.py
r4487 r4490 1 from ndgUtils. models.utilities import wrapGetText1 from ndgUtils.lib.utilities import wrapGetText 2 2 class AccessControl: 3 3 -
exist/trunk/python/ndgUtils/models/Atom.py
r4488 r4490 16 16 from ndgUtils.eXistConnector import eXistConnector 17 17 from ndgUtils.ETxmlView import et2text 18 from utilities import getTripleData, escapeSpecialCharacters, \19 tidyUpParameters, getISO8601Date 18 from ndgUtils.lib.utilities import getTripleData, escapeSpecialCharacters, \ 19 tidyUpParameters, getISO8601Date, normaliseLongitude 20 20 from ndgUtils.vocabtermdata import VocabTermData as VTD 21 21 from ndgUtils.models import MolesEntity as ME … … 31 31 logging.error(msg) 32 32 Exception.__init__(self, msg) 33 34 35 class ValidationError(Exception):36 """37 Exception handling for validation.38 """39 def __init__(self, errorDict):40 msg = "Data validation error"41 logging.error(msg)42 Exception.__init__(self, msg)43 for val in errorDict.itervalues():44 logging.error(val)45 self._errorDict = errorDict46 47 def unpack_errors(self):48 return self._errorDict49 33 50 34 … … 555 539 for relatedLink in self.relatedLinks: 556 540 if relatedLink.hasValue(): 541 import pdb 542 pdb.set_trace() 557 543 root.append(relatedLink.toXML()) 558 544 … … 579 565 author = Person() 580 566 author.name = self.ME.providerID 581 author.uri = self.ME.providerID567 #author.uri = self.ME.providerID 582 568 self.author = author 583 569 … … 674 660 self.content = [] 675 661 for content_line in content.split('\n'): 676 self.content.append( escapeSpecialCharacters(content_line))662 self.content.append(content_line) 677 663 678 664 Content = property(fset=__setContent, fget=__getContent, doc="Atom content") … … 705 691 summary = tree.findtext('summary') 706 692 if summary: 707 self.Summary = summary 693 self.Summary = summary#.decode('unicode_escape') 708 694 709 695 authorElement = tree.find('author') … … 903 889 ''' 904 890 logging.info("Adding spatial data to Atom") 905 bbox = ET.SubElement(element, "georss:where")906 891 if not self.minX: 907 892 logging.info("No spatial data specified") 908 893 return 909 894 bbox = ET.SubElement(element, "georss:where") 910 895 envelope = ET.SubElement(bbox, "gml:Envelope") 911 896 lc = ET.SubElement(envelope, "gml:lowerCorner") … … 1012 997 logging.debug("Returning matched links") 1013 998 return matchingLinks 1014 1015 1016 def validate(self):1017 '''1018 Check the various values of the various atom attributes; if an error with any of1019 these is found, raise a ValueError1020 @raise ValueError: if any atom attributes have a problem1021 '''1022 logging.info("Validating the atom data model")1023 errors = {}1024 if not self.title:1025 errors['title'] = "Title attribute cannot be empty"1026 1027 if self.minX or self.maxX or self.minY or self.maxY:1028 missingVals = False1029 incorrectFormat = False1030 for val in [self.minX, self.maxX, self.minY, self.maxY]:1031 if val == '':1032 missingVals = True1033 else:1034 try:1035 float(val)1036 except:1037 incorrectFormat = True1038 1039 if missingVals or incorrectFormat:1040 errors['spatialcoverage'] = ""1041 if missingVals:1042 errors['spatialcoverage'] += "Incomplete spatial coverage data.\n"1043 if incorrectFormat:1044 errors['spatialcoverage'] += "Spatial coverage data not in numerical format."1045 1046 if self.t1 or self.t2:1047 timeErrors = ''1048 d1 = None1049 d2 = None1050 if self.t1:1051 try:1052 d1 = datetime.datetime.strptime(self.t1, self.YEAR_FORMAT)1053 except:1054 timeErrors += "Incorrect start date format - '%s' - c.f. '2008-04-12. \n'" %self.t11055 if self.t2:1056 try:1057 d2 = datetime.datetime.strptime(self.t2, self.YEAR_FORMAT)1058 except:1059 timeErrors += "Incorrect end date format - '%s' - c.f. '2008-04-12. \n'" %self.t21060 1061 if d1 and d2:1062 if d1 > d2 or d2 < d1:1063 timeErrors += "Inconsistent date range - '%s' is not before '%s'" \1064 %(d1.strftime(self.YEAR_FORMAT), d2.strftime(self.YEAR_FORMAT))1065 1066 if timeErrors:1067 errors['temporalrange'] = timeErrors1068 1069 1070 # do a quick recursion over all the attributes to look for ascii characters1071 for key, val in self.__dict__.items():1072 if val:1073 if type(val) == str:1074 try:1075 # NB, the latin coding accepts unicode up to 2551076 correctedString = val.decode('latin-1')1077 except:1078 if not errors.has_key(key):1079 errors[key] = ''1080 errors[key] += "Illegal unicode found in string: '%s'.\n" %val1081 1082 if errors:1083 logging.warning("Errors found in atom data: %s" %errors)1084 raise ValidationError(errors)1085 logging.info("Atom model validated successfully")1086 999 1087 1000 … … 1156 1069 # Firstly, extract the bounding envelope 1157 1070 if bbox1: 1158 w, e = self.moveBox(bbox1[0],bbox1[2])1071 w, e = normaliseLongitude(bbox1[0],bbox1[2]) 1159 1072 n, s = (bbox1[3], bbox1[1]) 1160 1073 … … 1292 1205 dataArray.append(link) 1293 1206 logging.debug("Finished adding links") 1294 1295 1296 def moveBox(self, w,e):1297 ''' Take a 0,360 bounding box and force into -180,180 '''1298 ww,ee=float(w),float(e)1299 if ww<180.0 and ee>180.0:1300 return ww-180.0,ee-180.01301 else:1302 if ww>180.0:1303 return ww-360.,ee-360.1304 else:1305 return ww,ee -
exist/trunk/python/ndgUtils/models/DIF.py
r4487 r4490 2 2 # renderEntity etc ... 3 3 # 4 from ndgUtils. models.utilities import formatDateYYYY4 from ndgUtils.lib.utilities import formatDateYYYY 5 5 from ndgUtils.models.molesbounding import Bounding as Bounding 6 6 from People2 import * -
exist/trunk/python/ndgUtils/models/MolesEntity.py
r4444 r4490 14 14 from ndgUtils.elementtree import cElementTree as ET 15 15 import logging, datetime 16 from utilities import escapeSpecialCharacters16 from ndgUtils.lib.utilities import escapeSpecialCharacters 17 17 18 18 -
exist/trunk/python/ndgUtils/models/People.py
r4487 r4490 1 from ndgUtils. models.utilities import wrapGetText1 from ndgUtils.lib.utilities import wrapGetText 2 2 from ndgUtils.ETxmlView import nsdumb 3 3 from milk_server.lib.htmlUtilities import * -
exist/trunk/python/ndgUtils/models/existdbclient.py
r4483 r4490 31 31 inputs = {} 32 32 33 self.atomSchema = None 33 34 # NB, there are two routes through here: if a config file is specified 34 35 # without a hostname, the host will be taken to be the first entry in … … 36 37 if configFile: 37 38 if not self.eXistDBHostname: 38 self._ loadDBDetails(configFile)39 self.__loadDBDetails(configFile) 39 40 inputs['passwordFile'] = configFile 40 41 … … 47 48 48 49 # set up any collections required - NB, if these already exist they won't cause any files to be lost 49 self._setUpEXistAtomCollections() 50 self.__setUpEXistAtomCollections() 51 52 # add the schema required for atom validation 53 self.__addAtomSchema() 50 54 51 55 self.collections = None … … 54 58 55 59 logging.info("eXist DB connection initialised") 60 61 62 def __getSchema(self): 63 logging.debug("Getting atom schema data") 64 if not self.atomSchema: 65 self.atomSchema = ec.BASE_COLLECTION_PATH + \ 66 ndgXqueries.ATOM_MOLES_SCHEMA + '.xsd' 67 68 return self.atomSchema 69 70 AtomSchema = property(fget=__getSchema, doc="Atom schema path") 56 71 57 72 … … 69 84 70 85 71 def _setUpEXistAtomCollections(self): 86 def checkAtomSchemaCompliance(self, atomPath, atom = None): 87 ''' 88 Validate the specified atom in eXist with the atom schemae in eXist 89 @param atomPath: path to the atom in eXist 90 @keyword atom: if set to an atom, this will be created temporarily in eXist 91 - since it may not already exist there. Once validation is completed, the 92 file will be removed from eXist. 93 @return array: containing any errors found - NB, if an empty array is returned, 94 this indicates successful validation 95 ''' 96 logging.info("Validating atom, '%s' against schemae in eXist" %atomPath) 97 98 if atom: 99 logging.info("Creating temporary file in eXist to do validation against") 100 fileName = atom.datasetID + str(datetime.datetime.today().microsecond) 101 self.createEXistFile(atom.toPrettyXML(), \ 102 atom.getDefaultCollectionPath(), fileName) 103 atomPath = atom.getDefaultCollectionPath() + fileName 104 105 validationQuery = 'validation:validate-report("' + atomPath + \ 106 '", xs:anyURI("' + self.AtomSchema + '"))' 107 id, result = self.xmldb.executeQuery(validationQuery) 108 errorMessage = None 109 if result['hits'] == 0: 110 errorMessage = "Validation did not complete successfully - please retry" 111 elif result['hits'] > 1: 112 errorMessage = "More than one atom was validated - expecting only a single atom validation - please retry" 113 114 if atom: 115 logging.info("Deleting temporary file in eXist") 116 self.deleteEXistFile(atomPath) 117 118 if errorMessage: 119 logging.error(errorMessage) 120 raise SystemError(errorMessage) 121 122 doc = self.xmldb.retrieve(id, 0) 123 et = ET.fromstring(doc) 124 status = et.findtext('status') 125 126 # retrieve the error detail if invalid 127 errors = [] 128 if status == 'invalid': 129 logging.info("Atom is invalid - details as follows:") 130 for error in et.findall('message'): 131 lineNo = error.attrib.get('line') 132 colNo = error.attrib.get('column') 133 level = error.attrib.get('level') 134 repeat = error.attrib.get('repeat') 135 errorMessage = "%s at line %s, column %s: %s" %(level, lineNo, colNo, error.text) 136 if repeat: 137 errorMessage += " (%s times)" %repeat 138 errors.append(errorMessage) 139 logging.info(errorMessage) 140 else: 141 logging.info("Atom is valid") 142 143 logging.info("Validation complete") 144 return errors 145 146 147 def __setUpEXistAtomCollections(self): 72 148 ''' 73 149 Set up the required eXist collections needed for running the granulator script … … 86 162 87 163 88 def _loadDBDetails(self, configFile): 164 def __addAtomSchema(self): 165 ''' 166 Add the required atom schema to the atoms collection - to allow validation 167 of input atoms 168 ''' 169 logging.info("Adding atom schema to eXist") 170 xq = ndgXqueries() 171 schemae = [xq.ATOM_SCHEMA, xq.MOLES_SCHEMA, xq.ATOM_MOLES_SCHEMA] 172 for schema in schemae: 173 xml = xq.getSchema(schema) 174 self.createEXistFile(xml, ec.BASE_COLLECTION_PATH, schema + '.xsd') 175 logging.info("- schema added") 176 177 178 def __loadDBDetails(self, configFile): 89 179 ''' 90 180 Retrieve info from the eXist db config file … … 109 199 110 200 111 def _ lookupEXistFile(self, docPath):201 def __lookupEXistFile(self, docPath): 112 202 ''' 113 203 Look up a file in eXist using XPath … … 132 222 @return: contents of document if exists, None otherwise 133 223 ''' 134 id = self._ lookupEXistFile(docPath)224 id = self.__lookupEXistFile(docPath) 135 225 136 226 if not id: … … 150 240 logging.info("Checking if file, '%s', exists in eXist DB" %docPath) 151 241 152 id = self._ lookupEXistFile(docPath)242 id = self.__lookupEXistFile(docPath) 153 243 154 244 if id: … … 158 248 159 249 160 def _ addTimeStamp(self, fileName):250 def __addTimeStamp(self, fileName): 161 251 ''' 162 252 Add timestamp to input filename … … 199 289 200 290 # add timestamp to filename 201 fileName = self._ addTimeStamp(fileName)291 fileName = self.__addTimeStamp(fileName) 202 292 docPath = collection + fileName 203 293 -
exist/trunk/python/ndgUtils/models/molesbounding.py
r4487 r4490 34 34 for coverage in coverages: 35 35 self.set([helper.getText(coverage,i) for i in ebox[entity]]) 36 37 def geoString2float(self, x): 38 if x[-1:] in 'NE': 39 return float(x[:-1]) 40 elif x[-1:] in 'SW': 41 return -float(x[:-1]) 42 else: 43 return float(x) 36 44 37 45 def set(self,box): 38 46 try: 39 self.boxes.append([ geoString2float(i) for i in box])47 self.boxes.append([self.geoString2float(i) for i in box]) 40 48 self.nboxes+=1 41 49 except: -
exist/trunk/python/ndgUtils/models/stubB.py
r4487 r4490 8 8 # 9 9 10 from ndgUtils. models.utilities import wrapGetText10 from ndgUtils.lib.utilities import wrapGetText 11 11 from ndgUtils.BeautifulSoup import BeautifulSoup 12 12 from AccessControl import AccessControl -
exist/trunk/python/ndgUtils/ndgXqueries.py
r4483 r4490 7 7 programming languages). 8 8 ''' 9 # these are the different schemae needed to validate the atoms - NB, the main 10 # one is molesAtom1.0 - this imports the others 9 11 ATOM_MOLES_SCHEMA = "molesAtom1.0" 12 MOLES_SCHEMA = "moles2.0" 13 ATOM_SCHEMA = "atom1.0" 14 10 15 def __init__(self,directory='xquery'): 11 16 ''' Loads up xqueries from the internal package directory '''
Note: See TracChangeset
for help on using the changeset viewer.