source: TI01-discovery/trunk/ingestAutomation/OAIBatch/oaiClean.py @ 1599

#
# This code designed to clean up incoming harvested metadata documents for inclusion
# in the NDG eXist database ... 
# BNL, October 13, 2006
#

import os
try:
    #python 2.5
    import xml.etree.cElementTree as ET
except:
    try:
    #python 2.4, assumes you have it in the right sort of place ...
        import cElementTree as ET
    except:
        # and give up and raise an error if you haven't even got that ...
        import ElementTree as ET

def oaiClean(inputDirectoryName,outputDirectoryName,fileName,dcKeyWords=[]):
    
    ''' takes a file (fileName) in directory inputDirectoryName and cleans it up and writes
    it out in outputDirectoryName, with the following cleaning actions carried out:
        1) we strip namespaces,
        2) we add the ndg dcKeywords (a list), and 
        3) wrap it up in an instance of <ndgDoc> '''
        
    #construct input and output file identifiers
    fIn=os.path.join(inputDirectoryName,fileName)
    
    #is there any reason why it has to have a different name?
    fOut=os.path.join(outputDirectoryName,fileName)
    
    #load the file
    inTree=ET.parse(open(fIn))
    
    #create the output element and populate it with keywords
    outTree=ET.Element('ndgDoc')
    kws=ET.SubElement(outTree,'keywords')
    for keyword in dcKeyWords: 
        k=ET.SubElement(kws,'keyword')
        k.text=keyword
    doc=ET.SubElement(outTree,'doc')
    
    
    #get the root element of the input and clean it up ...
    root=inTree.getroot()
    
    # fix namespaces ... 
    newDoc=removeNS(root)
        
    #bif it in the output
    doc.append(newDoc)
    
    # this is wrong, we need to know what the input encoding is, but it'll do for now.
    # if we do need to muck with it, see http://effbot.org/zone/celementtree-encoding.htm
    # which also has links to methods of recognising the encoding ...
    output=ET.ElementTree(outTree)
    output.write(fOut,encoding='UTF-8')
    
def removeNS(e):
    ''' removes namespaces from element e ... the hardway, there must be an easier way '''
    tag=e.tag
    t=tag.split('}') # gets us the ET namespace definition split from the element tag
    if len(t)<>2: # no namespace
        y=ET.Element(t[0])
    else: # namespace exists
        y=ET.Element(t[1])
    y.text=e.text
    y.tail=e.tail
    for a in e.keys():
        # can be namespace bumf in attributes as well ... especially in the declarations
        aa=a.split('}')
        if len(aa)==1:
            y.attrib[a]=e.attrib[a]
        else:
            if a.find('schemaLocation')==-1: y.attrib[aa[1]]=e.attrib[a]
    for child in e:
        y.append(removeNS(child))
    return y
    

if __name__=="__main__":
    import sys
    try:
        indir,outdir,filename=sys.argv[1:4]
        if len(sys.argv)==5: 
            keywords=sys.argv[4].split(',')
        else:
            keywords=[]
    except:
        print sys.argv[1:]
        print "usage: oaiClean indirectory outdirectory filename keyword1,keyword2,... "
        exit()
    oaiClean(indir,outdir,filename,keywords)