source: TI01-discovery/trunk/ingestAutomation/OAIBatch/SpaceTimeIngestFromMOLES.py @ 1809

#!/usr/bin/env python
import cElementTree
import elementtree.ElementTree as etree
import molesReadWrite as MRW
import sys
import db_funcs
import os

#connect to db (in separate db functions module)
connection = db_funcs.db_connect()

def id_exists(Mid):
        sql = "select id from spatiotemp where id = '"+Mid+"';"
        cursor = connection.cursor()
        cursor.execute(sql)
        if len(cursor.fetchall()) <1:
            return False
        else:
            return True


def do_insert(Mid,west,south,east,north,startdate,enddate):
        sql = "INSERT INTO spatiotemp (id, coordinates, startdate, enddate) VALUES ( '"+Mid+ "', sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))', '"+startdate+"', '"+enddate+"');"
        print sql
        cursor = connection.cursor()
        cursor.execute(sql)
        connection.commit()

def do_update(Mid,west,south,east,north,startdate,enddate):
        sql = "UPDATE spatiotemp SET coordinates = sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))', startdate='"+startdate+"', enddate= '"+enddate+"' WHERE id='"+Mid+"';"
        print sql
        cursor = connection.cursor()
        cursor.execute(sql)     
        connection.commit()

def main(indir):
    if indir == "":
        sys.exit("Usage: argument 1 = full path of directory where MOLES records reside")
    else:
        print "INFO: moles records are in %s" %indir

#this is a fix to the  ElementTree namespace problem that namespaces are usually represented as ns0, ns1, ns2 etc.
    etree._namespace_map.update({'http://ndg.nerc.ac.uk/moles': 'moles', 'http://ndg.nerc.ac.uk/csml' : 'csml', 'http://www.w3.org/1999/xlink':'xlink'})
    numfilesproc = 0

    filenames = os.listdir(indir)
    for filename in filenames:
            if filename.find('.xml') != -1:
                full_filename = indir + "/" + filename
                dgMeta=MRW.dgMetadata()
                dgMeta.fromXML(cElementTree.ElementTree(file=full_filename).getroot())
                bbox=dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgSpatialCoverage.BoundingBox
                dates=dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgTemporalCoverage.DateRange
                coverage= [dates.DateRangeEnd, dates.DateRangeStart, bbox.LimitNorth, bbox.LimitSouth,bbox.LimitEast, bbox.LimitWest]
                #Mid = dgMeta.dgMetadataRecord.dgMetadataID.repositoryIdentifier+"__"+dgMeta.dgMetadataRecord.dgMetadataID.localIdentifier
                Mid = filename
                print coverage, Mid
                #parse the coordinates somewhat
                #west
                west = bbox.LimitWest.strip()
                if west.endswith('E'):
                    west=bbox.LimitWest.split('E')[0]
                elif west.endswith('W'):
                    if west.startswith('-'):
                        west = bbox.LimitWest.split('W')[0]
                    else:
                        west = "-" +bbox.LimitWest.split('W')[0]
                try:
                    float(west)
                except:
                    print "Error: Will not process File %s. Contains incorrect West bounding box limit." %full_filename
                    continue
                print "West = %s" %west
                #east
                east = bbox.LimitEast.strip()
                if east.endswith('E'):
                    east=bbox.LimitEast.split('E')[0]
                elif east.endswith('W'):
                    if east.startswith('-'):
                        east = bbox.LimitEast.split('W')[0]
                    else:
                        east = "-" +bbox.LimitEast.split('W')[0]
                try:
                    float(east)
                except:
                    print "Error: Will not process File %s. Contains incorrect East bounding box limit." %full_filename
                    continue
                print "East = %s" %east
                #north
                north = bbox.LimitNorth.strip()
                if north.endswith('N'):
                    north=bbox.LimitNorth.split('N')[0]
                elif north.endswith('S'):
                    if north.startswith('-'):
                        north = bbox.LimitNorth.split('S')[0]
                    else:
                        north = "-" +bbox.LimitNorth.split('S')[0]
                try:
                    float(north)
                except:
                    print "Error: Will not process File %s. Contains incorrect North bounding box limit." %full_filename
                    continue
                print "North = %s" %north
                #south
                south = bbox.LimitSouth.strip()
                if south.endswith('N'):
                    south=bbox.LimitSouth.split('N')[0]
                elif south.endswith('S'):
                    if south.startswith('-'):
                        south = bbox.LimitSouth.split('S')[0]
                    else:
                        south = "-" +bbox.LimitSouth.split('S')[0]
                try:
                    float(south)
                except:
                    print "Error: Will not process File %s. Contains incorrect North bounding box limit." %full_filename
                    continue
                print "North = %s" %south

                if id_exists( Mid ):
                        print "doc %s exists, updating\n" %Mid
                        do_update( Mid, west, south, east, north, dates.DateRangeStart, dates.DateRangeEnd )
                else:
                        print "doc %s does not exist, inserting new record\n" %Mid
                        do_insert( Mid, west, south, east, north, dates.DateRangeStart, dates.DateRangeEnd )

                numfilesproc += 1


    print 'SpaceTimeIngestFromMOLES.py ran to end. files processed= %s' %(numfilesproc)

if __name__=='__main__':
    indir=sys.argv[1]
    main(indir)