Changeset 8000

Timestamp:

02/12/11 14:47:11 (8 years ago)

Author:

sdonegan

Message:

Updated bad file handling and reporting

Location:

TI01-discovery-Ingest/trunk/v4.3.0/ingestAutomation-upgrade/OAIBatch

Files:

• PostgresDAO.py (modified) (3 diffs)
• abstractdocumentingester.py (modified) (12 diffs)

TI01-discovery-Ingest/trunk/v4.3.0/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

@@ -122 +122 @@
        
         self.getRecordID()
-        returnCode=0 # 0 if failed, 1 if update, 2 if create
-         
-        if self._record.db_id:               
-            if self.updateRecord():             
-                returnCode = 1
-        else:
+        returnCode=None # 0 if failed, 1 if update, 2 if create        
+        msg = None
+        
+        try:
+            if self._record.db_id:               
+                if self.updateRecord():                 
+                       returnCode = 1
+            else:
                 #create the record!          
-            if self.createRecord():
-                returnCode = 2
+                if self.createRecord():
+                       returnCode = 2
+                       
+        except Exception, error:
+            
+            returnCode = 0
+            
+            if (type(error).__name__ == 'UnicodeEncodeError') or (type(error).__name__ == 'UnicodeDecodeError'):
+                #get the position of the dodgy characters & message
+                convertMsg = 'Error with character encoding in this field (%s): %s'%(type(error).__name__,error)
+                logging.warn(convertMsg)
+                msg = convertMsg
+                
+            else:
+                logging.error("Error detected with parsing and inserting record into the database")
+                msg = "Possible error with sql content (bad characters?)"
                 
-        return returnCode
+        return returnCode, msg
             
 
@@ -514 +530 @@
         #Get rid of any xml comment characters - causing problems
         originalXMLdoc = re.sub("<!--.*-->","",self._record.originalFormat)
-        
-        
+                
         sqlCmd = "SELECT create_document('" + self._record.shortFilename + "', '" + \
             self._record.discovery_id + "', '" + self.dpwsID + "', '" + self._record.docType + "', '" + \
@@ -522 +537 @@
             self._record.dataset_name + "', '" + self._record.datacentre_name + "', '" + \
             self._record.dataset_lastEdit + "', '" + self._record.datasetStartNom + "', '" + self._record.datasetEndNom + "' );"
-        
         
         #sort out any nulls..

TI01-discovery-Ingest/trunk/v4.3.0/ingestAutomation-upgrade/OAIBatch/abstractdocumentingester.py

@@ -24 +24 @@
 from SchematronValidate import SchematronValidate as Schematron
 
+class SqlException(Exception):pass
+
 class AbstractDocumentIngester(object):
         '''
@@ -209 +211 @@
                 
                 try:
-                        
+                        #import pdb
+                        #pdb.set_trace()
                         #if record is valid (vital elements checked in ExtractISO - validDoc is True if everything ok..
                         
@@ -217 +220 @@
                         
                         if self.isoDataModel.validDoc is True:                  
-                                                                
+                                
                                 #record whats attempting to be ingested 
                                 record = PostgresRecord(filename, self._NDG_dataProvider,self._datacentre_groups, self._datacentre_namespace,
@@ -233 +236 @@
                                 # Finally, write the new record
                                 # 0 if failed, 1 if update, 2 if create                         
-                                returnCode = dao.createOrUpdateRecord()
+                                returnCode, msg = dao.createOrUpdateRecord()
+                                
+                                if returnCode == 0:
+                                        logging.error("Detected problem on update: msg")
+                                        
+                                        #most likely problem is a character encoding problem.. to get around this (& t be sure) sort the encoding of filename to be on the safe side:
+                                        datasetSafeName,cnvMsg = record.characterEncoding(self.isoDataModel.datasetName[0][0])
+                                        
+                                        self.ingestProblemMessage= "%s (Content problems: %s)" %(datasetSafeName,msg)
+                                        #import pdb; pdb.set_trace()
+                                        raise ValueError,self.ingestProblemMessage
+                                        #raise SqlException()
                                                                 
                                 if dao.processingWarning != '':
@@ -252 +266 @@
                                 raise ValueError,self.ingestProblemMessage
                         
-                except:
-                        
+                except Exception:
                         
                         #if error encountered, add to failure lisr
@@ -259 +272 @@
                         originalRecordFilename = self.inputFileOrigFinal[shortFilename]
                         self.updateFailList.append(originalRecordFilename)
+                        
                         
                         logging.error("Exception thrown - detail: ")
@@ -274 +288 @@
                                         logging.error(sys.exc_info())
                                         logging.error("NB, this record will need to be cleared manually from DB to ensure all relevant data is ingested")
-
+                        #import pdb;
+                        #pdb.set_trace()
                         self._no_problem_files += 1
                         
@@ -834 +849 @@
                         if self.ingestProcessID is not None and self.procID is None:
                                 sqlStatusCmd = "select update_ingest_info (%s, ' (ingesting %s of %s)');" %(self.ingestProcessID,counter,numfilesproc)                  
+                                                                
                                 self.pgc_IngestLog.runSQLCommand(sqlStatusCmd)
                         
@@ -843 +859 @@
                                 #record all problem messages
                                 #ingestProbMsg = ingestProbMsg + self.ingestProblemMessage + "\n"
+                                
                                 if self.ingestProblemMessage != '':
-                                        ingestProbMsg = ingestProbMsg + self.ingestProblemMessage                                       
-                                        self.problemMessageList[self.originalsDiscoveryFilesMap[filename]] = self.ingestProblemMessage
+                                        ingestProbMsg = ingestProbMsg + self.ingestProblemMessage
+                                        #Need to record datasetname here as well as the filename - filenames are usually useless (i.e. CSW harvest)
+                                        #key = '(%s(%s)' %(self.originalsDiscoveryFilesMap[filename], self.isoDataModel.datasetName[0][0])
+                                        key = self.originalsDiscoveryFilesMap[filename]
+                                        self.problemMessageList[key] = self.ingestProblemMessage
                                         
                                 else:
@@ -922 +942 @@
                 if len(self.updateFailList) > 0:
                         message = message +  "\nBelow are comments recorded on problem files...\n\n"
-                                        
-                        for badFile in self.updateFailList:
+                        
+                        for badFile in self.problemMessageList.keys(): #self.updateFailList:
                                 #recOpFile.write("PROBLEM_FILE " + badFile + "\n")
                                 
                                 #NOTE: there may not be a problem message for every file that failed to ingest..
-                                if badFile in self.problemMessageList.keys():
-                                        message = message +"\nPROBLEM_FILE " + badFile + "\n"
-                                        message = message + self.problemMessageList[badFile] + "\n"
+                                #if badFile in self.problemMessageList.keys():
+                                message = message +"\nPROBLEM_FILE " + badFile + "\n"
+                                message = message + self.problemMessageList[badFile] + "\n\n"
                                 
                 
@@ -952 +972 @@
                         if self._no_problem_files > 0:
                                 logging.info("There are %s problem files - recording these in stats logging db" %self._no_problem_files)
+                                
                                 
                                 #note sometimes message will not always be recorded, so just record name of file
@@ -971 +992 @@
                                         
                         #if process id has been supplied, then update the logging db with a "start_ingest" (only for DPWS ops)
+                        #import pdb; pdb.set_trace()
                         if self.procID is None:
                                 sqlStatusCmd = "select update_ingest_status (%s, 'end_ingest');" %self.ingestProcessID