Changeset 3972

Timestamp:

03/06/08 13:47:43 (11 years ago)

Author:

cbyrom

Message:

Use the short filename in the postgres DB for storing the original
document filename.
Add fix to allow proper handling of scope fields as a ts_vector.
Add TODO comments to highlight areas of concern + update docs.

Location:

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch

Files:

• PostgresDAO.py (modified) (4 diffs)
• PostgresRecord.py (modified) (7 diffs)
• README.txt (modified) (1 diff)
• oai_document_ingester.py (modified) (1 diff)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

@@ -227 +227 @@
         '''
         logging.info("Inserting new original document in Postgres DB")
-        sqlCmd = "SELECT create_document('" + self._record.filename + "', '" + \
+        sqlCmd = "SELECT create_document('" + self._record.shortFilename + "', '" + \
             self._record.discovery_id + "', '" + self._record.docType + "', '" + \
             self._record.originalFormat + "', '" + self._record.getAuthorsInfo() + "', '" + \
@@ -257 +257 @@
         logging.info("Updating original document in Postgres DB")
         sqlCmd = "SELECT update_document('" + str(self._record.db_id) + "', '" + \
-            self._record.filename + "', '" + \
+            self._record.shortFilename + "', '" + \
             self._record.discovery_id + "', '" + self._record.docType + "', '" + \
-            self._record.originalFormat + "', '" + self._record.getAuthorsInfo + "', '" + \
-            self._record.getParametersInfo + "', '" + self._record.getScopeInfo + "', '" + str(self._record.scn) + "');"
+            self._record.originalFormat + "', '" + self._record.getAuthorsInfo() + "', '" + \
+            self._record.getParametersInfo() + "', '" + self._record.getScopeInfo() + "', '" + str(self._record.scn) + "');"
         db_funcs.runSQLCommand(self._connection, sqlCmd)
         
@@ -272 +272 @@
         Insert the metadata docs into the postgres DB
         '''
-        logging.info("Inserting transformed documents for original document, %s, in Postgres DB", self._record.filename)
+        logging.info("Inserting transformed documents for original document, %s, in Postgres DB", self._record.shortFilename)
         if self._record.db_id is None:
             logging.info("No DB ID for the original record exists; cannot add associated transformed docs")
@@ -293 +293 @@
         Update the metadata docs into the postgres DB
         '''
-        logging.info("Updating transformed documents for original document, %s, in Postgres DB", self._record.filename)
+        logging.info("Updating transformed documents for original document, %s, in Postgres DB", self._record.shortFilename)
         if self._record.db_id is None:
             logging.info("No DB ID for the original record exists; cannot update associated transformed docs")

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

@@ -65 +65 @@
         tmp = filename.split('/')
         self._dir = '/'.join(tmp[0:len(tmp)-1])
-        self._shortFilename = tmp[len(tmp)-1]
+        self.shortFilename = tmp[len(tmp)-1]
         
         # dir to store a temp copy of the moles file, when produced - for use by other transforms
@@ -103 +103 @@
         '''
         return re.sub(r'\'', '\\\'', inputString)
+
+
+    def unescapeSpecialCharacters(self, inputString):
+        '''
+        Adjust the input string to remove escaped characters that would interfere with string or DB
+        operations
+        @param inputString: string to correct
+        @return: corrected string 
+        '''
+        str = re.sub(r'%20', ' ', inputString)
+        return 
     
     
@@ -129 +140 @@
             self.doMolesTransform()
             
-        self._fileUtils.createFile(self._molesDir + self._shortFilename, self._molesFormat)
+        self._fileUtils.createFile(self._molesDir + self.shortFilename, self._molesFormat)
         logging.info("Moles file created - at %s" %self._molesDir)
         
         # now load this moles file, for use when parsing out spatiotemporal, author and parameters data later on        
-        molesFile = self._molesDir + self._shortFilename
+        molesFile = self._molesDir + self.shortFilename
         logging.info('Retrieving spatiotemporal info from moles file, %s' %molesFile)
         
@@ -381 +392 @@
         authors = ""
         try:
-            creators = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataRoles.dgDataCreator
+            # TODO: check this is the correct path for author data - NB, this is not obvious from example files
+            # nb, if this is correct, need to escape out the %20 and %3 characters else it doesn't work - see unescape.. fn
+            creators = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataRoles.dgDataCreator.dgRoleHolder.dgMetadataID.localIdentifier
             logging.info("Found creator information - adding this to authors record")
             
@@ -397 +410 @@
             logging.info(detail)
             logging.info("- this suggests document does not contain cited author information.")
-            
+        
         self.authors = authors + " " + creators
         return self.authors
@@ -413 +426 @@
         params = ""
         try:
-            parameters = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgParameterSummary
-            logging.info("Found parameter information - adding this to record")
+            # TODO: check this is the correct path for parameters data - NB, this is not obvious from example files
+            parameters = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgParameterSummary.dgStdParameterMeasured
+            parameters_list = self.listify(parameters)
+            for parameter in parameters_list:
+                if parameters.dgValidTerm:
+                    logging.info("Found parameter information - adding this to record")
+                    params += " " + parameters.dgValidTerm
+            
             
         except Exception, detail:
@@ -450 +469 @@
             logging.info(detail)
             logging.info("- this suggests document does not contain scope information.")
-        
-        self.scope = scope
+
+        # NB, to_tsvector will remove any underscores -leading to, e.g. NERC_DDC becoming tokenised as 'NERC' and 'DDC'
+        # - to avoid this, use the following delimiter
+        self.scope = re.sub(r'_', 'UNDERSCORE', scope)
         return self.scope
             

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/README.txt

@@ -138 +138 @@
 the logger.  Ideally the logger would be used throughout code, but this is probably unlikely to happen.
 
+8. The 'author' and 'parameter' elements from the moles files are extracted by the PostgresRecord class.  The original code used
+a simple xpath search operator (&=) to find the data below a particular parent element.  Where possible, the more exact location
+has been identified and used in the new code, however, not all the required info has been available to test - in particular, the
+following need to be checked and adjusted if pointing to the incorrect field:
+
+  creators = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataRoles.dgDataCreator.dgRoleHolder.dgMetadataID.localIdentifier
+  authors = self.dgMeta.dgMetadataRecord.dgMetadataDescription.abstract.abstractOnlineReference.dgCitation.authors
+  parameters = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgParameterSummary.dgStdParameterMeasured.dgValidTerm
+  
+
+
+

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

@@ -327 +327 @@
                 
                 this_backupdir = backupdir_base + "_originals/"
-                fileUtils.makeBackUp(originals_dir, this_backupdir)
+                #fileUtils.makeBackUp(originals_dir, this_backupdir)
                 
                 #Clear out the original harvest records area and discovery dir
                 fileUtils.cleanDir(originals_dir)
-                fileUtils.cleanDir(discovery_dir)
+                #fileUtils.cleanDir(discovery_dir)
                 
                 logging.info("oai_document_ingest processing complete:")
                 if self._no_problem_files == 0:
                         logging.info("All files successfully processed - cleaning harvest directory")
-                        fileUtils.cleanDir(self._harvest_home)
+                        #fileUtils.cleanDir(self._harvest_home)
                 else:
                         logging.error("Problems experienced with %s files" %self._no_problem_files)