source: MILK/trunk/milk_server/milk_server/controllers/atom_editor/editatom.py @ 4491

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/MILK/trunk/milk_server/milk_server/controllers/atom_editor/editatom.py@4491
Revision 4491, 24.0 KB checked in by cbyrom, 12 years ago (diff)

Create new class, AtomValidator?, to act as a utility class for the Atom class - allowing validation of the following data:
i) External links

ii) Vocab data
iii) Schema compliance
iv) unicode compliance - with utf-8 encoding
v) data consistency within the atom data model


Store an instance in a global MILK variable for easy re-use by the atom-editor. Also, remove redundant geoUtilities class.

Line 
1'''
2 Class representing pylons controller for the creation and editing of atom
3 data
4 
5 @author: C Byrom, Tessella Sep 2008
6'''
7import logging, traceback, sys
8from paste.request import parse_querystring
9from xml.parsers.expat import ExpatError
10from formencode import Invalid
11from genshi.filters import HTMLFormFiller
12from genshi import HTML
13from milk_server.lib.base import *
14from milk_server.models.form import *
15from milk_server.lib import mailer
16from milk_server.lib.ndgInterface import interface
17import milk_server.lib.htmlUtilities as utils
18from milk_server.lib.atomutilities import savePageAndRender
19from ndgUtils import ndgObject
20from ndgUtils.models.Atom import Atom, Person, Link, Category
21from ndgUtils.lib.atomvalidator import ValidationError
22import ndgUtils.models.existdbclient as edc
23from ndgUtils.models.MolesEntity import MolesEntity as ME
24from ndgUtils.lib.utilities import escapeSpecialCharacters
25from ndgUtils.vocabtermdata import VocabTermData as VTD
26from granulatorTool.granulite import granulite as granulite
27
28class EditatomController(BaseController):
29    '''
30    Provides the pylons controller for editing NDG Atom documents.
31    '''
32    ADD_ASSOCIATIONS = 3
33    REMOVE_ASSOCIATIONS = 4
34   
35    def __setup(self,uri=None):
36        ''' Common setup stuff for all the actions on this controller '''
37        logging.info("Setting up EditatomController")
38        self.cf=request.environ['ndgConfig']
39
40        if uri:
41            try:
42                self.ndgObject = ndgObject(uri, config=self.cf)
43            except ValueError,e:
44                return e
45
46        self.inputs=dict(parse_querystring(request.environ))
47
48        logging.info("EditatomController set up")
49        return 0
50
51
52    def __handleError(self, e, template='error'):
53        '''
54        Handle exceptions thrown; if debug mode on, display full stack trace
55        in output, otherwise just show basic error message in error template
56        @param e: Exception to process
57        @keyword template: template to render - 'error' is the default - NB, if
58        an alternative is specified it should have a div with class set to 'error'
59        containing the variable, c.xml to display properly
60        '''
61        errorMessage = e.message
62        if g.debugAtomEditor:
63            errorMessage = traceback.format_exc()#print_exception(*sys.exc_info())
64
65        c.xml = 'Unexpected error loading page [%s]' %str(errorMessage)
66        c.doc=''
67        logging.error(c.xml)
68       
69        response.status_code = 400
70        return render("genshi", template)
71
72   
73    def upload(self, uri):
74        '''
75        Upload a CSML or granulite file and store it in the session variable
76        '''
77        logging.info("Uploading file...")
78        file = request.POST.get('upload_CSML')
79        isGranulite = False
80        if file is None:
81            # check for granulite
82            file = request.POST.get('upload_granulite')
83            isGranulite = True
84           
85        if file == '':
86            errorMessage = "Error: could not load file - please try again"
87            logging.error(errorMessage)
88            c.errors = {'Load error' : errorMessage}
89            return hc.index(hc())
90
91        logging.debug("- file name: '%s'" %file.name)
92        # Prepare the basic data model
93        c.errors = {}
94        try:
95            # NB, if loading a granulite, this will create the displayed atom
96            if uri != None:
97                self.prepareDataModel(uri)
98            else:
99                self.__setup()
100                c.atom = Atom()
101        except SystemError, e:
102            return self.__handleError(e)
103
104        # now process the input file and add any extra required data
105        try:
106            if isGranulite:
107                self.__processGranuliteFile(file.value)
108            else:
109                self.__processCSMLFile(file)
110        except Exception, e:
111            return self.__handleError(e, template='atom_editor/atom_granulator')
112
113        # save new data
114        self.saveAtomToExist(c.atom)
115                   
116        # now do redirection - NB, this ensures that current atom contents are
117        # reloaded and displayed
118        logging.info("File data loaded and extracted to atom")
119        h.redirect_to(controller = 'atom_editor/editatom', action='edit', \
120                        uri = c.atom.ndgURI)
121
122
123    def __processCSMLFile(self, file):
124        '''
125        Accept the contents of a CSML file and extract and add appropriate data
126        to the current atom data model
127        @param fileContents: contents of the file uploaded
128        '''
129        logging.info("Extracting CSML data")
130        c.atom.addCSMLData(file)
131        logging.info("Finished extracting CSML data")
132   
133
134    def __processGranuliteFile(self, fileContents):
135        '''
136        Accept the contents of a granulite file and extract and add appropriate data
137        to the current atom data model
138        @param fileContents: contents of the file uploaded
139        '''
140        logging.info("Processing granulite data")
141        # check for uploaded CSML/CDML file data
142        cdmlFile = request.POST.get('upload_cdml')
143        csmlFile = request.POST.get('upload_csml')
144        if cdmlFile and csmlFile:
145            raise ValueError("Cannot specify both CDML and CSML file - please choose a single one to ingest.")
146       
147        # NB, we'll be creating the atom in the default local eXist
148        eXistClient = self.__getExistClient('local')
149        gran = granulite(fileContents, eXistClient = eXistClient, \
150                         cdmlFile = cdmlFile, csmlFile = csmlFile)
151       
152        logging.info("Finished processing granulite data")
153
154   
155    def saveAtom(self, uri, saveLevel=0):
156        '''
157        Save the atom contents - NB, validation is done by method decoration
158        - if this fails, the action is reran as a GET with htmlfill auto-populating
159        the fields to keep them as they were before the submission
160        '''
161        logging.info("Saving input atom data")
162        c.errors = {}
163        try:
164            self.prepareDataModel(uri)
165        except SystemError, e:
166            return self.__handleError(e)
167       
168        inputs = request.params
169
170        # save atom association changes
171        if int(saveLevel) == self.ADD_ASSOCIATIONS:
172            atomLinks = self.extractAtomAssociations(inputs)
173            c.atom.addUniqueRelatedLinks(atomLinks)
174        elif int(saveLevel) == self.REMOVE_ASSOCIATIONS:
175            atomLinks = self.extractAtomAssociations(inputs)
176            c.atom.removeRelatedLinks(atomLinks)
177        else:
178            authors = self.extractAuthorDetails(inputs)
179            if authors:
180                c.atom.addAuthors(authors)
181   
182            onlineRefs = self.extractOnlineReferenceDetails(inputs)
183            c.atom.addOnlineReferences(onlineRefs)
184
185            params = self.extractParameterDetails(inputs)
186            # NB, the atom type and subtype are added to the categories when the
187            # atom is exported to XML - so don't need to worry about overwriting
188            # them now
189            c.atom.parameters = params
190           
191            if inputs.get('subtype'):
192                c.atom.subtype = self.getLatestTermURLFromDropDownInput( \
193                        inputs.get('subtype'))
194                c.atom.subtypeID = c.atom.subtype.split('/')[-1]
195
196        logging.info("Validating input")
197        try:
198        #    validator = AtomFormSchema()
199        #    validator.to_python(inputs)
200            g.validator.setAtom(c.atom)
201            g.validator.validateAtom()
202            #c.atom.validate()
203            logging.info("- input valid")
204        except ValidationError, e:
205            c.errors = e.unpack_errors()
206            logging.info("- input invalid")
207            return self.edit(uri)
208
209        self.saveAtomToExist(c.atom)
210                   
211        # now do redirection - NB, this ensures that current atom contents are
212        # reloaded and displayed
213        h.redirect_to(controller = 'atom_editor/editatom', action='edit', \
214                        uri = c.atom.ndgURI)
215
216
217    def prepareDataModel(self, uri):
218        '''
219        Set up the underlying atom data model - loading the bulk from eXist
220        then updating any input fields appropriately
221        '''
222        logging.info("Preparing underlying data model")
223        status=self.__setup(uri=uri)
224        if status:
225            c.xml='<p>%s</p>'%status
226            response.status_code = 400
227            raise SystemError('Problem experienced setting up data model')
228
229        logging.info("Retrieving document to edit")
230        # NB, don't use the cache as docs are likely to change
231        # quite a lot during editing; ensure you've always got
232        # the latest updates loaded
233        status,x = interface.GetXML(uri, useCache=False)
234
235        if not status:
236            code=400
237            if x.startswith('<p> Access Denied'):
238                code=401
239
240            c.xml='%s'%x
241            response.status_code = code
242            raise SystemError('Problem experienced retrieving atom doc from eXist')
243
244        # NB, passing in the inputs will overwrite any original values with the
245        # user input ones
246        inputs = request.params
247        c.atom = Atom(xmlString=str(x), ndgObject = self.ndgObject, **dict(inputs))
248       
249        # save the current atom - to avoid this needing be recreated by the
250        # asynch viewDeployments call
251        session['currentAtom'] = c.atom
252        session.save()
253        logging.info("Data model set up")
254
255
256    def prepareEditForm(self, uri):
257        '''
258        Get everything set up for displaying the edit form
259        @param uri: ndg url for the atom to load in edit mode
260        '''
261        try:
262            # NB, can get here directly from saveAtom - if there have been errors
263            # - in this case keep original data
264            if not c.atom:
265                self.prepareDataModel(uri)
266        except SystemError, e:
267            return self.__handleError(e)
268
269        c.title='Editing [%s]'%self.ndgObject
270        c.uri = c.atom.ndgURI
271       
272        c.saveLink = h.url_for(controller='atom_editor/editatom',action='saveAtom', \
273                               saveLevel='1',  uri = c.atom.ndgURI)
274        c.saveLink2 = h.url_for(controller='atom_editor/editatom',action='saveAtom', saveLevel='2')
275        c.saveAssoc = h.url_for(controller='atom_editor/editatom',action='saveAtom', \
276                                 saveLevel = self.REMOVE_ASSOCIATIONS)
277       
278        # adjust atom type to cope with activity deployment exception
279        atomType = c.atom.atomTypeID
280        if atomType == g.vtd.ACTIVITY_TERM and \
281            c.atom.subtypeID == g.vtd.DEPLOYMENT_TERM:
282            atomType = g.vtd.DEPLOYMENT_TERM
283       
284        c.addEntityLink = h.url_for(controller='atom_editor/listatom',action='list', searchData = '0', \
285                               associatedAtomID = c.atom.ndgURI, \
286                               associatedAtomType = atomType, 
287                               associationType = utils.ENTITY_ASSOCIATION)
288           
289        c.addGranuleLink = h.url_for(controller='atom_editor/listatom',action='list', searchData = '0', \
290                               associatedAtomID = c.atom.ndgURI, \
291                               associatedAtomType = atomType, 
292                               associationType = utils.GRANULE_ASSOCIATION)
293           
294        c.addDeploymentLink = h.url_for(controller='atom_editor/listatom',action='list', searchData = '0', \
295                               associatedAtomID = c.atom.ndgURI, \
296                               associatedAtomType = atomType, 
297                               associationType = utils.DEPLOYMENT_ASSOCIATION)
298
299        # account for special case where we're dealing with deployments
300        listVals = g.vtd.getValidSubTypes(c.atom.atomTypeID)
301        if c.atom.isDeployment():
302            listVals = [g.vtd.TERM_DATA[g.vtd.DEPLOYMENT_TERM]]
303
304        c.subTypes = utils.getVocabTermDataDropdown(listVals, \
305                                        selected=c.atom.subtype)
306       
307       
308        self.addRelatedLinksDropDowns()
309
310   
311    def edit(self, uri):
312        '''
313        Edit the specified uri
314        '''
315        logging.info("Setting up atom edit template")
316        self.prepareEditForm(uri)
317
318        try:
319            return savePageAndRender(self.pathInfo, "atom_editor/atom_editor")
320       
321        except ExpatError, e:
322            c.xml='XML content is not well formed'
323            c.doc=str(x)
324            logging.error("Error retrieving [%s] - XML content: %s" % (uri, e))
325
326        except Exception, e:
327            #we may be showing an xml document ... but it could go wrong if
328            #we have crap content ...
329            c.xml='Unexpected error [%s] viewing [%s]'%(str(e), uri)
330            c.doc=''
331            logging.error(c.xml)
332       
333        response.status_code = 400
334        return render("genshi", 'atom_editor/error')
335
336
337    def addRelatedLinksDropDowns(self):
338        '''
339        Set up the drop down lists required for the selection of online ref links
340        '''
341        # at the very least, we need a simple drop down list with no preselected
342        # values
343        logging.debug("Setting up drop down lists for related links")
344        c.relatedLinkTerms = utils.getVocabTermDataDropdown(\
345                g.vtd.getValidTypes(g.vtd.ONLINE_REF_CATEGORY))
346       
347        c.relatedLinkSelectedLists = {}
348        for link in c.atom.relatedLinks:
349            logging.debug("Adding dropdown for related link, '%s'" %(str(link)))
350            c.relatedLinkSelectedLists[str(link)] = \
351                utils.getVocabTermDataDropdown(g.vtd.getValidTypes(g.vtd.ONLINE_REF_CATEGORY), \
352                                                     selected=link.rel)
353
354        logging.debug("Finished setting up drop down lists")
355
356
357    def extractAuthorDetails(self, inputs):
358        '''
359        Retrieve author data from inputs and set appropriately on Atom, if any
360        found
361        @return: list of Person objects with the author data
362        '''
363        logging.info("Extracting author data from inputs")
364        processedAuthors = []
365        authors = []
366        for key in inputs:
367            keyBits = key.split('.')
368            if len(keyBits) == 3 and keyBits[1] not in processedAuthors:
369               
370                authorType = -1
371                if key.lower().startswith('author'):
372                    authorType = Person.AUTHOR_TYPE
373                elif key.lower().startswith('contributor'):
374                    authorType = Person.CONTRIBUTOR_TYPE
375                elif key.lower().startswith('responsible'):
376                    authorType = Person.RESPONSIBLE_PARTY_TYPE
377                else:
378                    continue
379
380                # NB, adding an empty object here is valid as it will clear out
381                # existing entries, potentially
382                author = Person(personType = authorType)
383                # check if the remove checkbox has been set
384                keyStem = ".".join(keyBits[0:2])
385                if inputs.get(keyStem + ".remove"):
386                    logging.info("Removing author data")
387                else:
388                    author.name = inputs.get(keyStem + '.name') or ""
389                    author.uri = inputs.get(keyStem + '.uri') or ""
390                    author.role = inputs.get(keyStem + '.role') or ""
391                   
392                    logging.info("Adding new author info")
393                    logging.debug("Extracted author (type:'%s', name:'%s', uri:'%s', role:'%s')" \
394                                  %(author.type, author.name, author.uri, author.role))
395                    authors.append(author)
396                processedAuthors.append(keyBits[1])
397
398        logging.info("Finished extracting author data")
399        return authors
400
401
402    def extractOnlineReferenceDetails(self, inputs):
403        '''
404        Retrieve online reference data from inputs and set appropriately on Atom, if any
405        found
406        @return: list of Link objects containing the extracted data
407        '''
408        logging.info("Extracting related links data from inputs")
409        processedLinks = []
410        links = []
411
412        for key in inputs:
413            keyBits = key.split('.')
414            if len(keyBits) == 3 and keyBits[1] not in processedLinks:
415               
416                if key.lower().startswith(Atom.ONLINE_REF_LABEL):
417                    link = Link()
418                    keyStem = ".".join(keyBits[0:2])
419                   
420                    if inputs.get(keyStem + ".remove"):
421                        logging.info("Removing online reference data")
422                    else:
423                        # NB, this is in the format vocabURL--termID, so requires further
424                        # processing
425                        link.rel = self.getLatestTermURLFromDropDownInput(inputs.get(keyStem + '.rel'))
426                        link.href = inputs.get(keyStem + '.href') or ""
427                        link.title = inputs.get(keyStem + '.title') or ""
428                           
429                        logging.info("Adding new online reference info")
430                        logging.debug("Extracted online reference (href:'%s', title:'%s', rel:'%s')" \
431                                      %(link.href, link.title, link.rel))
432                        links.append(link)
433
434                    processedLinks.append(keyBits[1])
435                else:
436                    continue
437
438        logging.info("Finished extracting links data")
439        return links
440
441
442    def extractParameterDetails(self, inputs):
443        '''
444        Retrieve parameters data from inputs and set appropriately on Atom, if any
445        found
446        @return: list of Category objects containing the extracted data
447        '''
448        logging.info("Extracting parameters data from inputs")
449        processedParameters = []
450        parameters = []
451
452        for key in inputs:
453            keyBits = key.split('.')
454            if len(keyBits) == 3 and keyBits[1] not in processedParameters:
455               
456                if key.lower().startswith(Atom.PARAMETER_LABEL):
457                    parameter = Category()
458                    keyStem = ".".join(keyBits[0:2])
459                   
460                    if inputs.get(keyStem + ".remove"):
461                        logging.info("Removing parameters data")
462                    else:
463                        parameter.term = inputs.get(keyStem + '.term') or ""
464                        parameter.scheme = inputs.get(keyStem + '.scheme') or ""
465                        parameter.label = inputs.get(keyStem + '.label') or ""
466                           
467                        logging.info("Adding new parameter info")
468                        logging.debug("Extracted parameter (vocabURL:'%s', label:'%s', term:'%s')" \
469                                      %(parameter.scheme, parameter.label, parameter.term))
470                        parameters.append(parameter)
471
472                    processedParameters.append(keyBits[1])
473                else:
474                    continue
475
476        logging.info("Finished extracting parameters data")
477        return parameters
478
479
480    def extractAtomAssociations(self, inputs):
481        '''
482        Retrieve atom data from inputs and create related links pointing to
483        this data
484        @return: list of Links representing the related atoms
485        '''
486        logging.info("Extracting related atom ID data from inputs")
487        atoms = []
488        processedAtoms = []
489
490        for key in inputs:
491            if key.lower().startswith(Atom.ATOM_REF_LABEL):
492                (x, href, title, rel) = key.split(Atom.DELIMITER)
493                # NB, we handle removes by effectively ignoring them later on
494                if href not in processedAtoms:
495                    processedAtoms.append(href)
496
497                    link = Link()
498                    link.href = href or ""
499                    link.title = title or ""
500                    link.rel = rel or ""
501                   
502                    logging.debug("Extracted atom info (href:'%s', title:'%s', rel:'%s')" \
503                                  %(link.href, link.title, link.rel))
504                    atoms.append(link)
505            else:
506                continue
507
508        logging.info("Finished extracting atoms data")
509        return atoms
510               
511
512    def getLatestTermURLFromDropDownInput(self, inputVal):
513        '''
514        Term ID and vocabURL are specified in the drop down menus
515        - using the input from this, return the lastest full href to the
516        term ID
517        '''
518        termData = inputVal.split('--')
519        return g.vtd.getCurrentVocabURI(termData[0]) + \
520                        "/" + termData[1]
521
522
523    def saveAtomToExist(self, atom):
524        '''
525        Save the specified atom in eXist
526        @param atom: atom object to save to eXist
527        @return atom: atom object saved in eXist
528        '''
529        logging.info("Saving changes to eXist")
530        self.eXist = self.__getExistClient(atom.ME.providerID)
531        createdAtom = self.eXist.createAtomInExist(atom)
532        logging.info("Changes successfully saved to eXist")
533        return createdAtom
534
535    def __getExistClient(self, providerID):
536        '''
537        Use the config data to set up and return an eXist client
538        '''
539        logging.info("Setting up eXist client")
540        # lookup the eXist DB to use according to the provider ID for the
541        # data - NB, this is specified in the ndgDiscovery.config file
542        existHost = self.cf.get('NDG_EXIST', providerID)
543        configFile = self.cf.get('NDG_EXIST','passwordFile')
544        eXistClient = edc.eXistDBClient(eXistDBHostname = existHost, \
545                                       configFile = configFile)
546        logging.info("Returning eXist client")
547        return eXistClient
548   
549    def create(self, saveData = None, **inputs):
550        '''
551        Create a new atom
552        '''
553        if saveData:
554            logging.info("Validating input")
555            try:
556                inputs = request.params
557                validator = CreateAtomFormSchema()
558                validator.to_python(inputs)
559                logging.info("- input valid")
560               
561                logging.info("Creating basic atom")
562                self.__setup()
563                atomTypeID = inputs.get('atomTypeID').split('--')[1]
564                inputs['atomTypeID'] = atomTypeID
565   
566                # activity deployments should have subtype deployment specified automatically
567                if atomTypeID == g.vtd.ACTIVITY_DEPLOYMENT_TERM:
568                    inputs['subtypeID'] = g.vtd.DEPLOYMENT_TERM
569                    inputs['atomTypeID'] = g.vtd.ACTIVITY_TERM
570                   
571                inputs['providerID'] = inputs.get('providerID').split('--')[1]
572                atom = self.saveAtomToExist(Atom(**dict(inputs)))
573               
574                h.redirect_to(controller = 'atom_editor/editatom', action='edit',
575                               uri = atom.ndgURI)
576            except Invalid, e:
577                c.errors = e.unpack_errors()
578               
579           
580        logging.info("Setting up atom create template")
581        c.title='Create new atom'
582       
583        # set up the drop down content - NB, add special case, 'deployment activity'
584        # - this is just a specialised activity - i.e. with subtype preset
585        deploymentActivity = g.vtd.TERM_DATA[g.vtd.ACTIVITY_DEPLOYMENT_TERM]
586        c.atomTypes = utils.getVocabTermDataDropdown(g.vtd.getValidTypes(g.vtd.ATOM_CATEGORY),
587                                               defaultVal = deploymentActivity, \
588                                               selected = inputs.get('atomTypeID'))
589        c.providerIDs = utils.getVocabTermDataDropdown(
590                            g.vtd.getValidTypes(g.vtd.PROVIDER_CATEGORY), 
591                            selected = inputs.get('providerID'))
592
593        try:
594            return savePageAndRender(self.pathInfo, 'atom_editor/atom_creator')
595
596        except Exception, e:
597            return self.__handleError(e)
598
599   
600    def createGranule(self, **inputs):
601        '''
602        Create a new atom from a granulite file
603        '''
604        logging.info("Setting up create atom from granulite template")
605        c.title='Create new data granule atom - from a granulite file'
606       
607        try:
608            return savePageAndRender(self.pathInfo, 'atom_editor/atom_granulator')
609
610        except Exception, e:
611            return self.__handleError(e)
Note: See TracBrowser for help on using the repository browser.