source: mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py @ 8182

'''
Created on 15 Nov 2011

@author: mnagni
'''
from ea_model.moles3_4.utilities.mo_responsiblepartyinfo import MO_ResponsiblePartyInfo
from ea_model.iso_19115_2006_metadata_corrigendum.reference_system_information.md_identifier import MD_Identifier
from httplib import HTTPConnection
from xml.etree.ElementTree import XML, tostring
import time, datetime
from ea_model.upcomingiso.ci_organisation import CI_Organisation
from ea_model.upcomingiso.ci_party import CI_Party
from ea_model.upcomingiso.ci_individual import CI_Individual
from libs.migration.exception.exceptions import NoDataLineage
from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_citation import CI_Citation
from ea_model.iso_19115_2006_metadata_corrigendum.data_quality_information.dq_element import DQ_Element
from ea_model.iso_19115_2006_metadata_corrigendum.data_quality_information.dq_conformanceresult import DQ_ConformanceResult
from hashlib import md5
from xml.sax.saxutils import unescape, escape
import html5lib
from html5lib import treebuilders
from ea_model.ceda_metadatamodel.ceda_result.ceda_result import CEDA_Result
from ea_model.moles3_4.result.mo_onlineresource import MO_OnlineResource
from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.url import URL
from ea_model.iso_19115_2006_metadata_corrigendum.extent_information.ex_geographicboundingbox import EX_GeographicBoundingBox
from ea_model.ceda_metadatamodel.ceda_computation.ceda_processing import CEDA_Processing
from ea_model.ceda_metadatamodel.ceda_acquisition.ceda_instrument import CEDA_Instrument
from ea_model.ceda_metadatamodel.ceda_observationprocess.ceda_compositeprocess import CEDA_CompositeProcess
from ea_model.ceda_metadatamodel.ceda_acquisition.ceda_acquisition import CEDA_Acquisition

base = '/exist/rest/atoms'

DS_pUBLISHED = 'published'
DS_WORKING = 'working'
DS_PUBLISHED = 'Published'
docStatus = (DS_pUBLISHED, DS_WORKING, DS_PUBLISHED)

DT_DEPLOYMENTS = 'deployments'
DT_DATA_ENTITIES = 'data_entities'
DT_DEPLOYMENT_DATA = 'deployment_data'
DT_DATA_GRANULES = 'data_granules'
docTypes = (DT_DEPLOYMENTS, DT_DATA_ENTITIES, DT_DEPLOYMENT_DATA, DT_DATA_GRANULES)

DO_BADC = 'badc.nerc.ac.uk'
DO_NEODC = 'neodc.nerc.ac.uk'
DO_UKSSDC = 'ukssdc.ac.uk'

CEDA = 'Centre for Environmental Data Archive'
docOwners = (DO_BADC, DO_NEODC, DO_UKSSDC)

atomNS = "{http://www.w3.org/2005/Atom}"
existNS = "{http://exist.sourceforge.net/NS/exist}"
molesNS = "{http://ndg.nerc.ac.uk/schema/moles2beta}"
htmlNS = "{http://www.w3.org/1999/xhtml}"
georssNS="{http://www.georss.org/georss/10}"
gmlNS="{http://www.opengis.net/gml}"
time_format = '%Y-%m-%dT%H:%M:%SZ'
ihost = 'bora.badc.rl.ac.uk'
iport = '8080'

#MD_Identifier codes
MD_CODE_MOLES2_CITATION = 'ceda_moles2_citation'

htmlParser = html5lib.HTMLParser(tree=treebuilders.getTreeBuilder("etree"), namespaceHTMLElements=False)

def calculateHash(text):
    """
        Returns an md5 hexadecimal representation of the given text
        @param text: the string to encode
        @return: the hexadecimal md5 value of the given text 
    """
    encoder = md5()
    encoder.update(text)
    return encoder.hexdigest()

def buildExistDocPath(docStatus, docType, docOwner, docName):
    '''
        @param docStatus: one value from commons.docStatus
        @param docType: one value from commons.docCollections 
        @param docOwner: one value from commons.docOwners
        @param docName: one value from commons.docOwners        
    '''        
    return '%s/%s' % (buildExistOwnerPath(docStatus, docType, docOwner), docName)

def buildExistOwnerPath(docStatus, docType, docOwner):
    '''
        @param docStatus: one value from commons.docStatus
        @param docType: one value from commons.docCollections 
        @param docOwner: one value from commons.docOwners
    '''        
    return '%s/%s' % (buildExistTypePath(docStatus, docType), docOwner)

def buildExistTypePath(docStatus, docType):
    '''
        @param docStatus: one value from commons.docStatus
        @param docType: one value from commons.docCollections 
    '''        
    return '%s/%s' % (buildExistStatusPath(docStatus), docType)

def buildExistStatusPath(docStatus):
    '''
        @param docStatus: one value from commons.docStatus 
    '''        
    return '/exist/rest/atoms/%s' % (docStatus)

def getAtomDocumentByMO(migrationObject):
    if migrationObject is None:
        raise Exception("migrationObject is None")
    mo_typeDict = {'DeploymentsMigration': DT_DEPLOYMENTS, 'DataEntityMigration': DT_DATA_ENTITIES, 'DeploymentDataMigration': DT_DEPLOYMENT_DATA}     
    return getAtomDocumentAsElementtree(migrationObject.doc_status, mo_typeDict[type(migrationObject).__name__], migrationObject.doc_owner, migrationObject.doc_name)

def getAtomDocumentHashByMO(migrationObject):
    if migrationObject is None:
        raise Exception("migrationObject is None")    
    mo_typeDict = {'DeploymentsMigration': DT_DEPLOYMENTS, 'DataEntityMigration': DT_DATA_ENTITIES, 'DeploymentDataMigration': DT_DEPLOYMENT_DATA}     
    text = _getAtomDocumentAsText(migrationObject.doc_status, mo_typeDict[type(migrationObject).__name__], migrationObject.doc_owner, migrationObject.doc_name)
    return calculateHash(text)

def getAtomDocumentHash(docStatus, docType, docOwner, docName):
    source = buildExistDocPath(docStatus, docType, docOwner, docName)
    text = _getDocument(source)
    return calculateHash(text)

def hasAtomDocumentSameHash(migrationObject):
    return getAtomDocumentHashByMO(migrationObject) == migrationObject.doc_hash

def getAtomDocumentByType(migrationObject, docType):
    if migrationObject is None:
        raise Exception("migrationObject is None")    
    return getAtomDocumentAsElementtree(migrationObject.doc_status, docType, migrationObject.doc_owner, migrationObject.doc_name)

def _getAtomDocumentAsText(docStatus, docType, docOwner, docName):
    source = buildExistDocPath(docStatus, docType, docOwner, docName)
    return _getDocument(source)

def getAtomDocumentAsElementtree(docStatus, docType, docOwner, docName):
    source = buildExistDocPath(docStatus, docType, docOwner, docName)
    return _getXMLDocument(source)

def _getXMLDocument(source):
    return XML(_getDocument(source))

def stringToTimestamp(timestring):
    return datetime.datetime.fromtimestamp(time.mktime(time.strptime(timestring, time_format)))

def _getDocument(source):
    conn = HTTPConnection(host = ihost, port = iport)
    conn.connect()
    req = conn.request('GET', source)
    res = conn.getresponse()
    xmlDoc = res.read()
    '''
    print (xmlDoc)
    '''
    conn.close()
    return xmlDoc

def _returnNotNoneText(element):
    if element is None:
        return None
    return element.text

def findMolesCreationDate(resourceXML):
    creationDate = resourceXML.find('%sentity/%smolesISO/%screated' % (molesNS, molesNS, molesNS))
    return _returnNotNoneText(creationDate)

def findMolesLineage(dataEntityMigration):
    resourceXML = getAtomDocumentByMO(dataEntityMigration)
    lineage = resourceXML.find('%sentity/%smolesISO/%slineage' % (molesNS, molesNS, molesNS))
    if lineage is None:
        raise NoDataLineage(dataEntityMigration)
    return lineage.text

def extractQuality(dataEntityMigration):
    resourceXML = getAtomDocumentByMO(dataEntityMigration)
    quality = resourceXML.find('%sentity/%smolesISO/%squality' % (molesNS, molesNS, molesNS))
    return _returnNotNoneText(quality)

def extractContent(dataEntityMigration):
    """
        Returns a dictionary containing the div composing the
        <content> element in a dataentity document.
    """
    resourceXML = getAtomDocumentByMO(dataEntityMigration)
    content = resourceXML.find('%scontent' % (atomNS))
    text = _returnNotNoneText(content)
    doc = htmlParser.parse(unescape(text))
    contentDict = {}
    for el in doc.findall('body//div'):    
        prop = el.get('property')
        if prop:
            if prop.startswith('cedacat'):
                contentDict[prop.split(':')[1]] = escape(tostring(el))
    return contentDict

def _extractAuthors(authorsCSV):
    if authorsCSV is None:
        return []
    authors = authorsCSV.split(',')
    for index in range(len(authors)):
        authors[index] = authors[index].strip()
        if len(authors[index]) == 0:
            authors.remove(authors[index])
    return authors

def findAuthorsInResource(resourceMigration):
    '''
        Returns a dictionary with the following keys:
        'authors': a list of string representing the authors
        'contributors': a list of string representing the contributors
    '''
    ret = {}   
    resourceXML = getAtomDocumentByMO(resourceMigration)
    ret['authors'] = findAuthorInResource(resourceXML)
    ret['contributors'] = findContributorInResource(resourceXML)
    return ret

def findAuthorInResource(resourceXML): 
    author = resourceXML.find('%sauthor/%sname' % (atomNS, atomNS))
    return _returnNotNoneText(author)

def findContributorInResource(resourceXML): 
    contributors = resourceXML.find('%scontributor/%sname' % (atomNS, atomNS))
    return _returnNotNoneText(contributors)

def findID(dataEntXML):
    ent_id = dataEntXML.find('%sid' % (atomNS))
    return _returnNotNoneText(ent_id)

def _extractLinks(dataEntXML, markers):
    linksDict = {}
    links = dataEntXML.findall('%slink' % (atomNS))
    for link in links:
        for marker in markers:
            _updateLinksDict(linksDict, link, marker)        
    return linksDict

def _extractMigrationObjectLinks(dataEntXML):
    markers = ['Deployment', 'DOWNLOAD', 'DOCUMENTATION', 'ACCESS', 'LOGO']
    return _extractLinks(dataEntXML, markers)

def _updateLinksDict(linksDict, link, linkMarker):
    if not linksDict.has_key(linkMarker):
        linksDict[linkMarker] = []
    rel = link.get('rel')
    if rel and rel.endswith('/' + linkMarker):
        linksDict[linkMarker].append({'href': link.get('href'), 'title': link.get('title')})

def _findLinksInMigrationDocument(dataEntityMigration):
    dataEntXML = getAtomDocumentByMO(dataEntityMigration)
    linksDict = _extractMigrationObjectLinks(dataEntXML)
    return linksDict

def findDownloadLinksInMigrationDocument(migrationObject):
    """
        Return a list of dictionaries describing a <link rel="...DOWNLOAD..."> tag type
        Each dictionary has two keys: 'href' and 'title'
        @param migrationObject: the migration instance to retrieve and parse
        @return: a list of dictionaries
    """
    linksDict = _findLinksInMigrationDocument(migrationObject)
    return linksDict['DOWNLOAD']

def findDOIInMigrationDocument(migrationObject):
    """
        Return a dictionary describing a <link rel="...DOCUMENTATION..."> tag type
        The dictionary has two keys: 'href' and 'title'
        @param migrationObject: the migration instance to retrieve and parse
        @return: a dictionary relative to the DOI, None otherwise
    """
    linksDict = _findLinksInMigrationDocument(migrationObject)
    for link in linksDict['DOCUMENTATION']:
        if link['href'].startswith('doi:'):
            return link
    return None

def findDeploymentsInDE(dataEntityMigration):
    linksDict = _findLinksInMigrationDocument(dataEntityMigration)
    links = _extractLinksByMarker(linksDict, 'Deployment')
    return [depName + '.atom' for depName in links]

def findSubTypeInDPT(resourceMigration):
    resourceXML = getAtomDocumentByMO(resourceMigration)
    categories = resourceXML.findall('%scategory' % (atomNS))
    for category in categories:
        if category.get("term") == "ATOM_SUBTYPE":
            return category.get("label")    
        
def extractTitle(deploymentMigration):
    resourceXML = getAtomDocumentByMO(deploymentMigration)
    title = resourceXML.find('%stitle' % (atomNS))
    return _returnNotNoneText(title)

def extractSummary(deploymentMigration, dataEntityMigration):
    resourceXML = getAtomDocumentByMO(deploymentMigration)
    summary = resourceXML.find('%ssummary' % (atomNS))
    ret = _returnNotNoneText(summary)
    if ret:
        return ret
    resourceXML = getAtomDocumentByMO(dataEntityMigration)
    summary = resourceXML.find('%ssummary' % (atomNS))
    return _returnNotNoneText(summary)

def extractGeographicExtentInMigrationDocument(migrationObject):
    """
        Extracts if existing the georss:where/gel:Enveloper/upper-lowerCorner elements.
        @param migrationObject: a migration object to retrieve to parse for data
        @return: None if no data are found, otherwise a dictionary with keys: 'east', 'north', 'west', 'south' where
        the values are float
    """
    resourceXML = getAtomDocumentByMO(migrationObject)
    upperCorner = resourceXML.find('%swhere/%sEnvelope/%supperCorner' % (georssNS, gmlNS, gmlNS))
    lowerCorner = resourceXML.find('%swhere/%sEnvelope/%slowerCorner' % (georssNS, gmlNS, gmlNS))
    ret = None
    if upperCorner != None and lowerCorner != None:
        upperCornerData = upperCorner.text.split()
        lowerCornerData = lowerCorner.text.split()
        ret = {'east': float(upperCornerData[0]), 'north': float(upperCornerData[1]), 'west': float(lowerCornerData[0]), 'south': float(lowerCornerData[1])}
    return ret

def findLinksInDeployment(deploymentMigration):
    """
        Returns a dictionary of links owned by the given dataEntity document
        @param deploymentMigration: a DeploymentMigration instance
        @return: a dictionary of links. The possible keys are ['ACTIVITY', 'DPT', 'OBS']
    """
    resourceXML = getAtomDocumentByMO(deploymentMigration)
    links = {}
    markers = ['ACTIVITY', 'DPT', 'OBS'] 
    linksDict = _extractLinks(resourceXML, markers)
    for marker in markers:    
        links[marker] = _extractLinksByMarker(linksDict, marker)
    return links

def _extractLinksByMarker(linksDict, marker):
    dpt = []
    if linksDict.has_key(marker):
        for link in linksDict[marker]:
            try:
                linkLongName = link['href'].split('/')[-1]
                linkName = linkLongName.rsplit('__ATOM__')[1]
                dpt.append(linkName)
            except Exception as ex:
                print ex
    return dpt


def getResourceRefs(deploymentRefs):
    '''
        Returns a list of Elements representing the inner resource reference items
        @param resourceRefs: the name of the eXist collection name below the 'deployments' one
    '''  
    XMLDepl = _getXMLDocument(deploymentRefs)
    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS))

def getOwnerRefs(docStatus, docType, docOwner):
    '''
        Returns a list of Elements representing the inner resource reference items
        @param resourceRefs: the name of the eXist collection name below the 'deployments' one
    '''      
    XMLDepl = _getXMLDocument(buildExistOwnerPath(docStatus, docType, docOwner))
    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS))

def getTypeRefs(docStatus, docType):
    '''
        Returns a list of Elements representing the inner resource reference items
        @param resourceRefs: the name of the eXist collection name below the 'deployments' one
    '''      
    XMLDepl = _getXMLDocument(buildExistTypePath(docStatus, docType))
    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS))



def getCollectionRefs(publishedRefs):
    '''
        Returns a list of Elements representing the inner deployment reference items
        @param basePublished: the name of the eXist collection name below the 'published' one
    '''  
    XMLPubl = _getXMLDocument(publishedRefs)
    return XMLPubl.findall('%scollection/%scollection' % (existNS, existNS))

def getResource(source, resourceName):
    resourceSource = '%s/%s' % (source, resourceName)
    resourceDoc = _getDocument(resourceSource)
    return XML(resourceDoc)

def createMO_ResponsiblePartyInfoAsCI_Organization(role, names):
    '''
        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty
        @param names: the name assigned to each CI_Party
    '''
    return createMO_ResponsiblePartyInfo(role, names, CI_Organisation)

def createMO_ResponsiblePartyInfoAsCI_Individual(role, names):
    '''
        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty
        @param names: the name assigned to each CI_Party
    '''
    return createMO_ResponsiblePartyInfo(role, names, CI_Individual)

def createMO_ResponsiblePartyInfo(role, names, partyType = CI_Party):
    """
        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty
        @param names: the name assigned to each CI_Party
        @param partyType: the CI_Party implementation to use 
    """
    mo_responsableInfo = MO_ResponsiblePartyInfo()
    mo_responsableInfo.role = role
    test = mo_responsableInfo.party
    parties = []
    for name in names:
        ci_org = partyType()
        ci_org.name = name
        parties.append(ci_org)
    mo_responsableInfo.party = parties
    return mo_responsableInfo

def createCI_Citation(title = ""):
    ci_citation = CI_Citation()
    ci_citation.title = title
    return ci_citation

def createMD_Identifier(title = "", code = ""):
    """
        Creates a new MD_Identifier
        @param title: the MD_Identifier.authority.title field
        @param code: the MD_Identifier.code field
    """
    md_identifier = MD_Identifier()
    md_identifier.code = code
    md_identifier.authority = createCI_Citation(title)
    return md_identifier

def createMO_OnlineResource(linkage, name = None, function = None, description = None, applicationProfile = None):
    """
        Creates a new CEDA_Result
        @param linkage: the MO_OnlineResource.linkage.url.??? field
        @param name: the MO_OnlineResource.linkage.name field
        @param function: the MO_OnlineResource.function field 
        @param description: the MO_OnlineResource.description field
        @param applicationProfile: the MO_OnlineResource.applicationProfile field
    """
    on_line_resource = MO_OnlineResource()
    url = URL()
    #url.???? = linkage    
    on_line_resource.linkage = url
    if name: 
        on_line_resource.name = name
    if function:
        on_line_resource.function = function
    if description:
        on_line_resource.description = description
    if applicationProfile:
        on_line_resource.applicationProfile = applicationProfile
    return on_line_resource        

def createCEDA_Result(linkage, name = None, function = None, description = None, applicationProfile = None):
    """
        Creates a new CEDA_Result
        @param linkage: the CEDA_Result.source.linkage.url.??? field
        @param name: the CEDA_Result.source.linkage.name field
        @param function: the CEDA_Result.source.function field 
    """
    ceda_result = CEDA_Result()
    on_line_resource = createMO_OnlineResource(linkage, name, function, description, applicationProfile)        
    ceda_result.source.append(on_line_resource)
    return ceda_result



def createDQ_ConformanceResult(explaination = ""):
    dq_conformanceResult = DQ_ConformanceResult()
    dq_conformanceResult.explanation = explaination
    return dq_conformanceResult

def createDQ_Element(explaination = ""):
    dq_element = DQ_Element()
    dq_element.result = []
    dq_result = createDQ_ConformanceResult(explaination)
    dq_element.result.append(dq_result)
    return dq_element

def createEX_GeographicBoundingBox(east, north, west, south):
    """
        Creates an EX_GeographicBoundingBox instance
        @param east: the eastBoundLongitude attribute as float
        @param north: the northBoundLongitude attribute as float
        @param west: the westBoundLongitude attribute as float
        @param south: the southBoundLongitude attribute as float                
    """
    ex_geographic_bb = EX_GeographicBoundingBox()
    ex_geographic_bb.eastBoundLongitude = east
    ex_geographic_bb.northBoundLatitude = north
    ex_geographic_bb.westBoundLongitude = west
    ex_geographic_bb.southBoundLatitude = south
    return ex_geographic_bb
    
def createCEDA_Processing():
    ceda_processing = CEDA_Processing()
    return ceda_processing


def createCEDA_Instrument():
    ceda_instrument = CEDA_Instrument()
    return ceda_instrument

def createCEDA_CompositeProcess():
    ceda_cp = CEDA_CompositeProcess()
    return ceda_cp

def createCEDA_Acquisition():
    ceda_acquisition = CEDA_Acquisition()
    return ceda_acquisition