Changeset 3862
- Timestamp:
- 07/05/08 17:16:18 (13 years ago)
- Location:
- TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py
r3860 r3862 190 190 191 191 # Work out the relationship between the spatial and temporal data and handle appropriately 192 timeData = self._record. stData.getTemporalData()193 spatialData = self._record. stData.getSpatialData()192 timeData = self._record.getTemporalData() 193 spatialData = self._record.getSpatialData() 194 194 195 195 # check if we have any spatiotemporal data to add; escape if not -
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py
r3860 r3862 74 74 self.scn = 1 # system change number - keeps track of number of mods to a particular row 75 75 76 # do some initial setting up of record 77 self.doRecordTransforms() 78 self.getSpatioTemporalData() 79 76 # spatiotemporal data object 77 self.stData = None 80 78 81 79 def escapeSpecialCharacters(self, inputString): … … 251 249 252 250 # the doc type doesn't exist - so run the xquery 253 setattr(self, attributeName, self.doTransform(xqName)) 251 transformedDoc = self.doTransform(xqName) 252 setattr(self, attributeName, transformedDoc) 253 return transformedDoc 254 254 255 255 … … 258 258 Return a list of all the available doc types in the record 259 259 ''' 260 if len(self._allDocs) > 0: 260 # if the stored docs array is the same size as the array of all doc types 261 # assume all transforms have been done - and just return these 262 if len(self._allDocs) == len(self.documentTypes): 261 263 return self._allDocs 262 264 263 265 for docType in self.documentTypes: 264 266 self._allDocs.append([docType, self.getDocumentFormat(docType)]) 267 265 268 return self._allDocs 269 270 271 def getTemporalData(self): 272 ''' 273 Retrieves the temporal data for the record; if this hasn't been discovered yet, 274 do the necessary parsing 275 @return: TimeRange object array with temporal data 276 ''' 277 if self.stData is None: 278 self.getSpatioTemporalData() 279 280 return self.stData.getTemporalData() 281 282 283 def getSpatialData(self): 284 ''' 285 Retrieves the spatial data for the record; if this hasn't been discovered yet, 286 do the necessary parsing 287 @return: Coords object array with spatial data 288 ''' 289 if self.stData is None: 290 self.getSpatioTemporalData() 291 292 return self.stData.getSpatialData() 266 293 267 294 -
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py
r3861 r3862 139 139 logging.info("Datacentre classified as an NDG data provider") 140 140 else: 141 logging.info("Datacentre is not classifi cied as an NDG data provider")142 print self.lineSeparator141 logging.info("Datacentre is not classified as an NDG data provider") 142 logging.info(self.lineSeparator) 143 143 144 144 … … 165 165 Display input params for the script 166 166 ''' 167 print "Usage: python oai_document_ingester.py -v|d<datacentre>"168 print " - where:\n <datacentre> is the data centre to ingest data from; and"169 print " -v - verbose mode for output logging"170 print " -d - debug mode for output logging"167 logging.info("Usage: python oai_document_ingester.py [OPTION] <datacentre>") 168 logging.info(" - where:\n <datacentre> is the data centre to ingest data from; and options are:") 169 logging.info(" -v - verbose mode for output logging") 170 logging.info(" -d - debug mode for output logging") 171 171 sys.exit(2) 172 172 … … 177 177 ''' 178 178 self.lineSeparator = "-----------------------------" 179 print self.lineSeparator180 print "RUNNING: oai_document_ingester.py"179 logging.info(self.lineSeparator) 180 logging.info("RUNNING: oai_document_ingester.py") 181 181 182 182 # check for verbose option … … 185 185 except getopt.GetoptError, err: 186 186 # print help information and exit: 187 print str(err) # will print something like "option -a not recognized" 188 self.usage() 187 logging.info(str(err)) # will print something like "option -a not recognized" 189 188 190 189 loggingLevel = logging.WARNING 191 190 for o, a in opts: 192 191 if o == "-v": 193 print " - Verbose mode ON"192 logging.info(" - Verbose mode ON") 194 193 loggingLevel = logging.INFO 195 194 elif o == "-d": 196 print " - Debug mode ON"195 logging.info(" - Debug mode ON") 197 196 loggingLevel = logging.DEBUG 198 197 … … 200 199 format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 201 200 202 print self.lineSeparator201 logging.info(self.lineSeparator) 203 202 204 203 if datacentre is None: … … 230 229 231 230 # check harvest dir exists and that there are any records to harvest? 232 dpath = os.path.normpath(os.path.dirname(self._harvest_home)) 233 if not os.path.exists(dpath): 231 if not os.path.exists(self._harvest_home): 234 232 logging.info("Harvest directory for datacentre %s (%s) could not be found - exiting" \ 235 233 %(datacentre, self._harvest_home)) … … 238 236 logging.info("Nothing to harvest this time from %s" %datacentre) 239 237 return 240 241 238 242 239 # The directory to put things for a tape backup (should already exist) … … 318 315 #fileUtils.cleanDir(self._harvest_home) 319 316 320 print self.lineSeparator321 print "INFO: Number of files processed = %s" %numfilesproc322 print "INFO: Number of files ingested = %s" %self._no_files_ingested317 logging.info(self.lineSeparator) 318 logging.info("INFO: Number of files processed = %s" %numfilesproc) 319 logging.info("INFO: Number of files ingested = %s" %self._no_files_ingested) 323 320 if status == 0: 324 print "INFO: Procedure oai_document_ingester.py completed"321 logging.info("INFO: Procedure oai_document_ingester.py completed") 325 322 else: 326 print "ERROR: Procedure oai_document_ingester.py FAILED with status %s" %status327 print self.lineSeparator323 logging.error("ERROR: Procedure oai_document_ingester.py FAILED with status %s" %status) 324 logging.info(self.lineSeparator) 328 325 329 326 330 327 if __name__=="__main__": 328 logging.basicConfig(level=logging.INFO, 329 format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 331 330 opts, args = getopt.getopt(sys.argv[1:], '-vd') 332 331 if len(args) < 1: -
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py
r3861 r3862 9 9 10 10 lineSeparator = "-----------------------------" 11 print lineSeparator12 print "RUNNING: run_all_ingest.py"13 print lineSeparator14 11 15 12 # configure logging 16 13 logging.basicConfig(level=logging.INFO, 17 14 format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 15 logging.info(lineSeparator) 16 logging.info("RUNNING: run_all_ingest.py") 17 logging.info(lineSeparator) 18 18 19 19 … … 42 42 error_counter +=1 43 43 44 print "run_all_ingest.py complete - processed %s config files" %run_counter 44 logging.info("run_all_ingest.py complete - processed %s config files" %run_counter) 45 45 if error_counter > 0: 46 print "WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter46 logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter) 47 47
Note: See TracChangeset
for help on using the changeset viewer.