Changeset 1797 for TI01-discovery

Timestamp:

01/12/06 18:26:25 (13 years ago)

Author:

selatham

Message:

Do a bit of lat/lon parsing. Actually ingests now

Location:

TI01-discovery/trunk/ingestAutomation/OAIBatch

Files:

• SpaceTimeIngestFromMOLES.py (modified) (5 diffs)
• oai_ingest.py (modified) (1 diff)

TI01-discovery/trunk/ingestAutomation/OAIBatch/SpaceTimeIngestFromMOLES.py

@@ -10 +10 @@
 
 def id_exists(Mid):
-        sql = "select id from spatio where id = '"+Mid+"';"
+        sql = "select id from spatiotemp where id = '"+Mid+"';"
         cursor = connection.cursor()
         cursor.execute(sql)
@@ -19 +19 @@
 
 
-def do_insert(Mid,west,south,east,north):
-        sql = "INSERT INTO spatio (id, coordinates) VALUES ( "+Mid+ ", sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))' );"
+def do_insert(Mid,west,south,east,north,startdate,enddate):
+        sql = "INSERT INTO spatiotemp (id, coordinates, startdate, enddate) VALUES ( '"+Mid+ "', sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))', '"+startdate+"', '"+enddate+"');"
+        print sql
         cursor = connection.cursor()
         cursor.execute(sql)
+        connection.commit()
 
-def do_update(Mid,west,south,east,north):
-        sql = "UPDATE spatio SET coordinates = sbox'(("+west+"d , "+south+"d), ("+east+"d , ".north+"d))' WHERE id="+Mid+";"
+def do_update(Mid,west,south,east,north,startdate,enddate):
+        sql = "UPDATE spatiotemp SET coordinates = sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))', startdate='"+startdate+"', enddate= '"+enddate+"' WHERE id='"+Mid+"';"
+        print sql
         cursor = connection.cursor()
-        cursor.execute(sql)
+        cursor.execute(sql)     
+        connection.commit()
 
-def main(args=None):
-    indir= sys.argv[1]
+def main(indir):
     if indir == "":
         sys.exit("Usage: argument 1 = full path of directory where MOLES records reside")
@@ -36 +39 @@
         print "INFO: moles records are in %s" %indir
     numfilesproc = 0
-
-
-    #does record already exist in db?
-    #select_string="select a.fld_data_provider_local_id from ndgb.tbl_data_entities as a where a.fld_data_provider_id='%s'" %repositoryID
-    #cursor     = connection.cursor()
-    #cursor.execute(select_string)
-    #all_ids = cursor.fetchall()
-    #reccount= len(all_ids)
 
     filenames = os.listdir(indir)
@@ -54 +49 @@
                 dates=dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgTemporalCoverage.DateRange
                 coverage= [dates.DateRangeEnd, dates.DateRangeStart, bbox.LimitNorth, bbox.LimitSouth,bbox.LimitEast, bbox.LimitWest]
-                Mid = dgMeta.dgMetadataRecord.dgMetadataID.repositoryIdentifier+"__"+dgMeta.dgMetadataRecord.dgMetadataID.localIdentifier
+                #Mid = dgMeta.dgMetadataRecord.dgMetadataID.repositoryIdentifier+"__"+dgMeta.dgMetadataRecord.dgMetadataID.localIdentifier
+                Mid = filename
                 print coverage, Mid
+                #parse the coordinates somewhat
+                #west
+                west = bbox.LimitWest.strip()
+                if west.endswith('E'):
+                    west=bbox.LimitWest.split('E')[0]
+                elif west.endswith('W'):
+                    if west.startswith('-'):
+                        west = bbox.LimitWest.split('W')[0]
+                    else:
+                        west = "-" +bbox.LimitWest.split('W')[0]
+                try:
+                    float(west)
+                except:
+                    print "Error: Will not process File %s. Contains incorrect West bounding box limit." %full_filename
+                    continue
+                print "West = %s" %west
+                #east
+                east = bbox.LimitEast.strip()
+                if east.endswith('E'):
+                    east=bbox.LimitEast.split('E')[0]
+                elif east.endswith('W'):
+                    if east.startswith('-'):
+                        east = bbox.LimitEast.split('W')[0]
+                    else:
+                        east = "-" +bbox.LimitEast.split('W')[0]
+                try:
+                    float(east)
+                except:
+                    print "Error: Will not process File %s. Contains incorrect East bounding box limit." %full_filename
+                    continue
+                print "East = %s" %east
+                #north
+                north = bbox.LimitNorth.strip()
+                if north.endswith('N'):
+                    north=bbox.LimitNorth.split('N')[0]
+                elif north.endswith('S'):
+                    if north.startswith('-'):
+                        north = bbox.LimitNorth.split('S')[0]
+                    else:
+                        north = "-" +bbox.LimitNorth.split('S')[0]
+                try:
+                    float(north)
+                except:
+                    print "Error: Will not process File %s. Contains incorrect North bounding box limit." %full_filename
+                    continue
+                print "North = %s" %north
+                #south
+                south = bbox.LimitSouth.strip()
+                if south.endswith('N'):
+                    south=bbox.LimitSouth.split('N')[0]
+                elif south.endswith('S'):
+                    if south.startswith('-'):
+                        south = bbox.LimitSouth.split('S')[0]
+                    else:
+                        south = "-" +bbox.LimitSouth.split('S')[0]
+                try:
+                    float(south)
+                except:
+                    print "Error: Will not process File %s. Contains incorrect North bounding box limit." %full_filename
+                    continue
+                print "North = %s" %south
 
                 if id_exists( Mid ):
                         print "doc %s exists, updating\n" %Mid
-                        do_update( Mid, bbox.LimitWest, bbox.LimitSouth, bbox.LimitEast, bbox.LimitNorth )
+                        do_update( Mid, west, south, east, north, dates.DateRangeStart, dates.DateRangeEnd )
                 else:
                         print "doc %s does not exist, inserting new record\n" %Mid
-                        do_insert( Mid, bbox.LimitWest, bbox.LimitSouth, bbox.LimitEast, bbox.LimitNorth )
+                        do_insert( Mid, west, south, east, north, dates.DateRangeStart, dates.DateRangeEnd )
 
                 numfilesproc += 1
@@ -70 +127 @@
 
 if __name__=='__main__':
-    main()
+    indir=sys.argv[1]
+    main(indir)

TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

@@ -173 +173 @@
 
 # Then run the minimum moles creator  which will run over all records in the supplied collection
-# Will it pass back records? or add staright to /db/discovery/moles?
-
-
+# Passes back records to supplied outdir with original name
+#runthething(indir,outdir,
+
+#Extract the spatiotemporal info from created moles and put in Postgres db
+outdir = "./testmoles"
+try:
+    SpaceTimeIngestFromMOLES.main(outdir)
+except:
+    print "SpaceTimeIngestFromMOLES failed. Carrying on to do backups"
 
 #Make copies of discovery and oai/originals areas to backup area for tape backups