Changeset 1797 for TI01-discovery
- Timestamp:
- 01/12/06 18:26:25 (13 years ago)
- Location:
- TI01-discovery/trunk/ingestAutomation/OAIBatch
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
TI01-discovery/trunk/ingestAutomation/OAIBatch/SpaceTimeIngestFromMOLES.py
r1794 r1797 10 10 11 11 def id_exists(Mid): 12 sql = "select id from spatio where id = '"+Mid+"';"12 sql = "select id from spatiotemp where id = '"+Mid+"';" 13 13 cursor = connection.cursor() 14 14 cursor.execute(sql) … … 19 19 20 20 21 def do_insert(Mid,west,south,east,north): 22 sql = "INSERT INTO spatio (id, coordinates) VALUES ( "+Mid+ ", sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))' );" 21 def do_insert(Mid,west,south,east,north,startdate,enddate): 22 sql = "INSERT INTO spatiotemp (id, coordinates, startdate, enddate) VALUES ( '"+Mid+ "', sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))', '"+startdate+"', '"+enddate+"');" 23 print sql 23 24 cursor = connection.cursor() 24 25 cursor.execute(sql) 26 connection.commit() 25 27 26 def do_update(Mid,west,south,east,north): 27 sql = "UPDATE spatio SET coordinates = sbox'(("+west+"d , "+south+"d), ("+east+"d , ".north+"d))' WHERE id="+Mid+";" 28 def do_update(Mid,west,south,east,north,startdate,enddate): 29 sql = "UPDATE spatiotemp SET coordinates = sbox'(("+west+"d , "+south+"d), ("+east+"d , "+north+"d))', startdate='"+startdate+"', enddate= '"+enddate+"' WHERE id='"+Mid+"';" 30 print sql 28 31 cursor = connection.cursor() 29 cursor.execute(sql) 32 cursor.execute(sql) 33 connection.commit() 30 34 31 def main(args=None): 32 indir= sys.argv[1] 35 def main(indir): 33 36 if indir == "": 34 37 sys.exit("Usage: argument 1 = full path of directory where MOLES records reside") … … 36 39 print "INFO: moles records are in %s" %indir 37 40 numfilesproc = 0 38 39 40 #does record already exist in db?41 #select_string="select a.fld_data_provider_local_id from ndgb.tbl_data_entities as a where a.fld_data_provider_id='%s'" %repositoryID42 #cursor = connection.cursor()43 #cursor.execute(select_string)44 #all_ids = cursor.fetchall()45 #reccount= len(all_ids)46 41 47 42 filenames = os.listdir(indir) … … 54 49 dates=dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgTemporalCoverage.DateRange 55 50 coverage= [dates.DateRangeEnd, dates.DateRangeStart, bbox.LimitNorth, bbox.LimitSouth,bbox.LimitEast, bbox.LimitWest] 56 Mid = dgMeta.dgMetadataRecord.dgMetadataID.repositoryIdentifier+"__"+dgMeta.dgMetadataRecord.dgMetadataID.localIdentifier 51 #Mid = dgMeta.dgMetadataRecord.dgMetadataID.repositoryIdentifier+"__"+dgMeta.dgMetadataRecord.dgMetadataID.localIdentifier 52 Mid = filename 57 53 print coverage, Mid 54 #parse the coordinates somewhat 55 #west 56 west = bbox.LimitWest.strip() 57 if west.endswith('E'): 58 west=bbox.LimitWest.split('E')[0] 59 elif west.endswith('W'): 60 if west.startswith('-'): 61 west = bbox.LimitWest.split('W')[0] 62 else: 63 west = "-" +bbox.LimitWest.split('W')[0] 64 try: 65 float(west) 66 except: 67 print "Error: Will not process File %s. Contains incorrect West bounding box limit." %full_filename 68 continue 69 print "West = %s" %west 70 #east 71 east = bbox.LimitEast.strip() 72 if east.endswith('E'): 73 east=bbox.LimitEast.split('E')[0] 74 elif east.endswith('W'): 75 if east.startswith('-'): 76 east = bbox.LimitEast.split('W')[0] 77 else: 78 east = "-" +bbox.LimitEast.split('W')[0] 79 try: 80 float(east) 81 except: 82 print "Error: Will not process File %s. Contains incorrect East bounding box limit." %full_filename 83 continue 84 print "East = %s" %east 85 #north 86 north = bbox.LimitNorth.strip() 87 if north.endswith('N'): 88 north=bbox.LimitNorth.split('N')[0] 89 elif north.endswith('S'): 90 if north.startswith('-'): 91 north = bbox.LimitNorth.split('S')[0] 92 else: 93 north = "-" +bbox.LimitNorth.split('S')[0] 94 try: 95 float(north) 96 except: 97 print "Error: Will not process File %s. Contains incorrect North bounding box limit." %full_filename 98 continue 99 print "North = %s" %north 100 #south 101 south = bbox.LimitSouth.strip() 102 if south.endswith('N'): 103 south=bbox.LimitSouth.split('N')[0] 104 elif south.endswith('S'): 105 if south.startswith('-'): 106 south = bbox.LimitSouth.split('S')[0] 107 else: 108 south = "-" +bbox.LimitSouth.split('S')[0] 109 try: 110 float(south) 111 except: 112 print "Error: Will not process File %s. Contains incorrect North bounding box limit." %full_filename 113 continue 114 print "North = %s" %south 58 115 59 116 if id_exists( Mid ): 60 117 print "doc %s exists, updating\n" %Mid 61 do_update( Mid, bbox.LimitWest, bbox.LimitSouth, bbox.LimitEast, bbox.LimitNorth)118 do_update( Mid, west, south, east, north, dates.DateRangeStart, dates.DateRangeEnd ) 62 119 else: 63 120 print "doc %s does not exist, inserting new record\n" %Mid 64 do_insert( Mid, bbox.LimitWest, bbox.LimitSouth, bbox.LimitEast, bbox.LimitNorth)121 do_insert( Mid, west, south, east, north, dates.DateRangeStart, dates.DateRangeEnd ) 65 122 66 123 numfilesproc += 1 … … 70 127 71 128 if __name__=='__main__': 72 main() 129 indir=sys.argv[1] 130 main(indir) -
TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py
r1788 r1797 173 173 174 174 # Then run the minimum moles creator which will run over all records in the supplied collection 175 # Will it pass back records? or add staright to /db/discovery/moles? 176 177 175 # Passes back records to supplied outdir with original name 176 #runthething(indir,outdir, 177 178 #Extract the spatiotemporal info from created moles and put in Postgres db 179 outdir = "./testmoles" 180 try: 181 SpaceTimeIngestFromMOLES.main(outdir) 182 except: 183 print "SpaceTimeIngestFromMOLES failed. Carrying on to do backups" 178 184 179 185 #Make copies of discovery and oai/originals areas to backup area for tape backups
Note: See TracChangeset
for help on using the changeset viewer.