source: TI01-discovery/trunk/ingestAutomation/OAIBatch/retired/oaiClean.py @ 3779

#
# This code designed to clean up incoming harvested metadata documents for inclusion
# in the NDG eXist database ... 
# BNL, October 13, 2006
# History:
#  SEL 23/11/2006  Make wrapping optional - so can just remove namespace

import os
try:
    #python 2.5
    import xml.etree.cElementTree as ET
except:
    try:
    #python 2.4, assumes you have it in the right sort of place ...
        import cElementTree as ET
    except:
        # and give up and raise an error if you haven't even got that ...
        import ElementTree as ET

def oaiClean(inputDirectoryName,outputDirectoryName,fileName,wrapFlag,dcKeyWords=[]):
    
    ''' takes a file (fileName) in directory inputDirectoryName and cleans it up and writes
    it out in outputDirectoryName, with the following cleaning actions carried out:
        1) we strip namespaces,
        Optionally
        2) we add the ndg dcKeywords (a list), and 
        3) wrap it up in an instance of <ndgDoc> '''
        
    #construct input and output file identifiers
    fIn=os.path.join(inputDirectoryName,fileName)
    
    #is there any reason why it has to have a different name?
    fOut=os.path.join(outputDirectoryName,fileName)
    
    #load the file
    inTree=ET.parse(open(fIn))
    
    #get the root element of the input and clean it up ...
    root=inTree.getroot()

    # fix namespaces ...
    newDoc=removeNS(root)


    #if wrapping is indicated, create a wrapper output element and populate it with keywords
    if wrapFlag==True:
        outTree=ET.Element('ndgDoc')
        kws=ET.SubElement(outTree,'keywords')
        for keyword in dcKeyWords:
                k=ET.SubElement(kws,'keyword')
                k.text=keyword
        doc=ET.SubElement(outTree,'doc')
        #bif it in the output
        doc.append(newDoc)
        # this is wrong, we need to know what the input encoding is, but it'll do for now.
        # if we do need to muck with it, see http://effbot.org/zone/celementtree-encoding.htm
        # which also has links to methods of recognising the encoding ...
        output=ET.ElementTree(outTree)
        output.write(fOut,encoding='UTF-8')
    else:
        # otherwise just put the cleaned-up original in the output
        output=ET.ElementTree(newDoc)
        output.write(fOut,encoding='UTF-8')



def removeNS(e):
    ''' removes namespaces from element e ... the hardway, there must be an easier way '''
    tag=e.tag
    t=tag.split('}') # gets us the ET namespace definition split from the element tag
    if len(t)<>2: # no namespace
        y=ET.Element(t[0])
    else: # namespace exists
        y=ET.Element(t[1])
    y.text=e.text
    y.tail=e.tail
    for a in e.keys():
        # can be namespace bumf in attributes as well ... especially in the declarations
        aa=a.split('}')
        if len(aa)==1:
            y.attrib[a]=e.attrib[a]
        else:
            if a.find('schemaLocation')==-1: y.attrib[aa[1]]=e.attrib[a]
    for child in e:
        y.append(removeNS(child))
    return y


if __name__=="__main__":
    import sys
    try:
        indir,outdir,filename,wrapFlag=sys.argv[1:5]
        if len(sys.argv)==6:
            keywords=sys.argv[5].split(',')
        else:
            keywords=[]
    except:
        print sys.argv[1:]
        print "usage: oaiClean indirectory outdirectory filename wrapFlag keyword1,keyword2,... "
        print "     : wrapFlag expects a boolean."
        exit()
    oaiClean(indir,outdir,filename,wrapFlag,keywords)