Changeset 3967 for TI01-discovery

Timestamp:

02/06/08 10:43:00 (11 years ago)

Author:

cbyrom

Message:

Update code to allow the extraction of authors, parameters and scope
from moles files + adjust the data model to handle these new data.

Location:

TI01-discovery/branches/ingestAutomation-upgrade

Files:

• OAIBatch/PostgresDAO.py (modified) (2 diffs)
• OAIBatch/PostgresRecord.py (modified) (6 diffs)
• database/ingest_procedures.sql (modified) (4 diffs)
• database/original_document.sql (modified) (2 diffs)
• database/spatial_data.sql (modified) (1 diff)
• database/test_data.sql (modified) (2 diffs)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

@@ -229 +229 @@
         sqlCmd = "SELECT create_document('" + self._record.filename + "', '" + \
             self._record.discovery_id + "', '" + self._record.docType + "', '" + \
-            self._record.originalFormat + "');" 
+            self._record.originalFormat + "', '" + self._record.getAuthorsInfo() + "', '" + \
+            self._record.getParametersInfo() + "', '" + self._record.getScopeInfo() + "');" 
 
         id = db_funcs.runSQLCommand(self._connection, sqlCmd)
@@ -258 +259 @@
             self._record.filename + "', '" + \
             self._record.discovery_id + "', '" + self._record.docType + "', '" + \
-            self._record.originalFormat + "', '" + str(self._record.scn) + "');"
+            self._record.originalFormat + "', '" + self._record.getAuthorsInfo + "', '" + \
+            self._record.getParametersInfo + "', '" + self._record.getScopeInfo + "', '" + str(self._record.scn) + "');"
         db_funcs.runSQLCommand(self._connection, sqlCmd)
         

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

@@ -1 +1 @@
 #!/usr/bin/env python
 '''
-Class representing the contents of a row in the metadata_record postgres DB table
+Class representing the a document to be ingested into the postgres DB table
 C Byrom Apr 08
 '''
@@ -23 +23 @@
 class PostgresRecord:
     ''' 
-    Class representing the contents of a row in the metadata_record postgres DB table
+    Class representing the a document to be ingested into the postgres DB table
     @param filename: Name of file to use a metadata record
-    @param 
+    @param ndg_dataprovider
+    @param datacentre_groups
+    @param datacentre_namespace
+    @param discovery_id
+    @param xq
+    @param doctype - type of doc to process
     '''
     # TODO MDIP transforms do not work very well for lots of files - so currently hiding these
     documentTypes = ['MOLES', 'DIF', 'DC', 'ISO19139']#, 'MDIP']
+    
+    # vocab server - used for finding scope values in the moles files
+    ndg_data_provider_vocab = "http://vocab.ndg.nerc.ac.uk/term/N010"
         
     def __init__(self, filename, ndg_dataprovider, datacentre_groups, datacentre_namespace, discovery_id, xq, docType):
@@ -61 +69 @@
         # dir to store a temp copy of the moles file, when produced - for use by other transforms
         self._molesDir = None
+        # object to hold the moles file - this will be loaded in when it is created - in order to extract
+        # spatiotemporal data, etc
+        self.dgMeta = None
 
         # firstly load contents of file
@@ -78 +89 @@
         # spatiotemporal data object
         self.stData = None
+        
+        # fields to hold author, parameter and scope data
+        self.authors = None
+        self.parameters = None
+        self.scope = None
 
     def escapeSpecialCharacters(self, inputString):
@@ -115 +131 @@
         self._fileUtils.createFile(self._molesDir + self._shortFilename, self._molesFormat)
         logging.info("Moles file created - at %s" %self._molesDir)
+        
+        # now load this moles file, for use when parsing out spatiotemporal, author and parameters data later on        
+        molesFile = self._molesDir + self._shortFilename
+        logging.info('Retrieving spatiotemporal info from moles file, %s' %molesFile)
+        
+        # load in the moles file and put this into an object for direct access to the xml elements
+        self.dgMeta=MRW.dgMetadata()
+        try:
+            self.dgMeta.fromXML(cElementTree.ElementTree(file=molesFile).getroot())
+        except Exception, detail:
+            raise SystemError, 'Cannot parse the XML moles document %s. Detail:\n%s' %(molesFile, detail)
+
             
 
@@ -313 +341 @@
         Extract spatio temporal data from the original document
         '''
+        logging.info('Retrieving spatiotemporal info from moles file')
         # initialise the various spatiotemporal arrays used to extract data to
         self.stData = SpatioTemporalData()
         
-        molesFile = self._molesDir + self._shortFilename
-        logging.info('Retrieving spatiotemporal info from moles file, %s' %molesFile)
-        
-        # load in the moles file and put this into an object for direct access to the xml elements
-        dgMeta=MRW.dgMetadata()
-        try:
-            dgMeta.fromXML(cElementTree.ElementTree(file=molesFile).getroot())
-        except Exception, detail:
-            raise SystemError, 'Cannot parse the XML moles document %s. Detail:\n%s' %(molesFile, detail)
-
+        if self.dgMeta is None:
+            self.createMolesFile()
+            
         # do quick checks to see if the relevant data exists
-        if not dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary:
+        if not self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary:
             logging.info("No data summary elements found - assuming no spatiotemporal data available")
             return
         
-        if not dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage:
+        if not self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage:
             logging.info("No data coverage elements found - assuming no spatiotemporal data available")
             return
         
-        if not dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgSpatialCoverage:
+        if not self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgSpatialCoverage:
             logging.info("No spatial coverage elements found - assuming no spatial data available")
         else:
-            self.getCoordData(dgMeta)
-
-        if not dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgTemporalCoverage:
+            self.getCoordData(self.dgMeta)
+
+        if not self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgTemporalCoverage:
             logging.info("No temporal coverage elements found - assuming no temporal data available")
         else:
-            self.getTimeRangeData(dgMeta)
-
-
+            self.getTimeRangeData(self.dgMeta)
+
+    
+    def getAuthorsInfo(self):
+        '''
+        Extract authors info from the moles file
+        '''
+        logging.info('Retrieving authors info from moles file')
+        
+        if self.dgMeta is None:
+            self.createMolesFile()
+            
+        logging.info("Extracting author info")
+        creators = ""
+        authors = ""
+        try:
+            creators = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataRoles.dgDataCreator
+            logging.info("Found creator information - adding this to authors record")
+            
+        except Exception, detail:
+            logging.info("Exception thrown whilst trying to find creator information:")
+            logging.info(detail)
+            logging.info("- this suggests document does not contain creator information.")
+
+        try:
+            authors = self.dgMeta.dgMetadataRecord.dgMetadataDescription.abstract.abstractOnlineReference.dgCitation.authors
+            logging.info("Found cited author information - adding this to authors record")
+            
+        except Exception, detail:
+            logging.info("Exception thrown whilst trying to find cited author information:")
+            logging.info(detail)
+            logging.info("- this suggests document does not contain cited author information.")
+            
+        self.authors = authors + " " + creators
+        return self.authors
+    
+    
+    def getParametersInfo(self):
+        '''
+        Extract parameters info from the moles file
+        '''
+        logging.info('Retrieving parameters info from moles file')
+        
+        if self.dgMeta is None:
+            self.createMolesFile()
+            
+        params = ""
+        try:
+            parameters = self.dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgParameterSummary
+            logging.info("Found parameter information - adding this to record")
+            
+        except Exception, detail:
+            logging.info("Exception thrown whilst trying to find parameter information:")
+            logging.info(detail)
+            logging.info("- this suggests document does not contain parameter information.")
+        
+        self.parameters = params
+        return self.parameters
+    
+    
+    def getScopeInfo(self):
+        '''
+        Extract scope info from the moles file
+        '''
+        logging.info('Retrieving scope info from moles file')
+        
+        if self.dgMeta is None:
+            self.createMolesFile()
+            
+        scope = ""
+        try:
+            keywords = self.dgMeta.dgMetadataRecord.dgStructuredKeyword
+            logging.info("Found keyword information - parsing this for scope")
+
+            keywords_list = self.listify(keywords)
+            for keyword in keywords_list:
+                if keyword.dgValidTermID:
+                    if keyword.dgValidTermID.ParentListID.strip().startswith(self.ndg_data_provider_vocab):
+                        logging.info("Found scope value - adding this to record")
+                        scope += " " + keyword.dgValidTerm.strip()
+            
+        except Exception, detail:
+            logging.info("Exception thrown whilst trying to find scope information:")
+            logging.info(detail)
+            logging.info("- this suggests document does not contain scope information.")
+        
+        self.scope = scope
+        return self.scope
+            
+            
     def getTimeRangeData(self, dgMeta):
         '''

TI01-discovery/branches/ingestAutomation-upgrade/database/ingest_procedures.sql

@@ -21 +21 @@
 
 DROP FUNCTION create_document(filename_in varchar(255), discovery_id_in varchar(255), 
-        doc_type_in text, original_document_in text) CASCADE;
+        doc_type_in text, original_document_in text, authors text, parameters text, scope text) CASCADE;
 CREATE FUNCTION create_document(filename_in varchar(255), discovery_id_in varchar(255), 
-        doc_type_in text, original_document_in text) RETURNS integer AS 
+        doc_type_in text, original_document_in text, authors text, parameters text, scope text) RETURNS integer AS 
 $$
         DECLARE
@@ -30 +30 @@
         -- This inserts a new document into the DB
         INSERT INTO ORIGINAL_DOCUMENT (original_document_id, original_document_filename, 
-            discovery_id, original_format, original_document, ts_vector, create_date, 
-                        harvest_count, scn) VALUES (
+            discovery_id, original_format, original_document, document_ts_vector, authors_ts_vector, 
+            parameters_ts_vector, scope_ts_vector, create_date, harvest_count, scn) VALUES (
             DEFAULT, filename_in, discovery_id_in, doc_type_in, original_document_in, 
-                        to_tsvector('english', original_document_in), current_timestamp, 1, 1);
+                        to_tsvector('english', original_document_in), to_tsvector('english', authors), 
+                        to_tsvector('english', parameters), to_tsvector('english', scope), current_timestamp, 1, 1);
 
                 SELECT original_document_id INTO db_id FROM ORIGINAL_DOCUMENT WHERE discovery_id = discovery_id_in;
@@ -56 +57 @@
 
 DROP FUNCTION update_document(original_document_id_in int, filename_in varchar(255), 
-        discovery_id_in varchar(255), doc_type_in text, original_document_in text, scn_in int) CASCADE;
+        discovery_id_in varchar(255), doc_type_in text, original_document_in text, 
+        authors text, parameters text, scope text, scn_in int) CASCADE;
 CREATE FUNCTION update_document(original_document_id_in int, filename_in varchar(255), 
-        discovery_id_in varchar(255), doc_type_in text, original_document_in text, scn_in int) 
+        discovery_id_in varchar(255), doc_type_in text, original_document_in text, 
+        authors text, parameters text, scope text, scn_in int) 
         RETURNS VOID AS 
 $$
@@ -79 +82 @@
                         original_format = doc_type_in,
                         original_document = original_document_in,
-            ts_vector = to_tsvector('english', original_document_in),
+            document_ts_vector = to_tsvector('english', original_document_in),
+            authors_ts_vector = to_tsvector('english', authors),
+            parameters_ts_vector = to_tsvector('english', parameters),
+            scope_ts_vector = to_tsvector('english', scope),
             update_date = current_timestamp, 
                         harvest_count = 1,

TI01-discovery/branches/ingestAutomation-upgrade/database/original_document.sql

@@ -12 +12 @@
                 original_format text,
                 original_document text,
-                ts_vector       tsvector,
+                document_ts_vector      tsvector,
+                authors_ts_vector       tsvector,
+                parameters_ts_vector    tsvector,
+                scope_ts_vector tsvector,
                 create_date     timestamp,
                 update_date timestamp,
@@ -29 +32 @@
 
 -- Create index on searchable column to speed up searches
-CREATE INDEX textsearch_idx ON original_document USING gin(ts_vector);
+CREATE INDEX textsearch_idx ON original_document USING gin(document_ts_vector);

TI01-discovery/branches/ingestAutomation-upgrade/database/spatial_data.sql

@@ -16 +16 @@
 -- add 2D geometry column, 'geometry', to table - with SRS val of 4326
 select addgeometrycolumn('spatial_data','geometry',4326,'GEOMETRY',2);
+
+-- Create index on searchable column to speed up searches
+CREATE INDEX spatialsearch_idx ON spatial_data USING GIST(geometry);

TI01-discovery/branches/ingestAutomation-upgrade/database/test_data.sql

@@ -5 +5 @@
  * 
  */
-INSERT INTO original_document VALUES (DEFAULT, 'tst.xml', 'test_dummy', 'DIF', null, null, current_timestamp, null, 1, 1);
+INSERT INTO original_document VALUES (DEFAULT, 'tst.xml', 'test_dummy', 'DIF', null, null, null, null, to_tsvector('NERC'), current_timestamp, null, 1, 1);
 INSERT INTO original_document VALUES (DEFAULT, 'tst1.xml', 'test_record', 'DIF', '<?xml version="1.0" encoding="utf-8"?>
 <kml xmlns="http://earth.google.com/kml/2.2">
@@ -28 +28 @@
   </Folder>
 </kml>',
-null, current_timestamp, null, 1, 1);
+null, null, null, to_tsvector('MDIP'), current_timestamp, null, 1, 1);
 
-INSERT INTO original_document VALUES (DEFAULT, 'tst2.xml', 'test_dummy1', 'DIF', 'freds freds bloo' , null, current_timestamp, null, 1, 1);
-INSERT INTO original_document VALUES (DEFAULT, 'tst3.xml', 'test_dummy2', null, 'fred fred bloo fred', null, current_timestamp, null, 1, 1);
-INSERT INTO original_document VALUES (DEFAULT, 'tst4.xml', 'test_dummy3', null, 'fred fred bloo, fred, fred, fred, fre, fred', null, current_timestamp, null, 1, 1);
+INSERT INTO original_document VALUES (DEFAULT, 'tst2.xml', 'test_dummy1', 'DIF', 'freds freds bloo' , null, null, null, to_tsvector('MDIP'), current_timestamp, null, 1, 1);
+INSERT INTO original_document VALUES (DEFAULT, 'tst3.xml', 'test_dummy2', null, 'fred fred bloo fred', null, null, null, to_tsvector('MDIP'), current_timestamp, null, 1, 1);
+INSERT INTO original_document VALUES (DEFAULT, 'tst4.xml', 'test_dummy3', null, 'fred fred bloo, fred, fred, fred, fre, fred', null, null, null, null, current_timestamp, null, 1, 1);
 
-UPDATE original_document SET ts_vector = to_tsvector('english', original_document);
+UPDATE original_document SET document_ts_vector = to_tsvector('english', original_document);
 
-select original_document_id, ts_vector, ts_rank(ts_vector, query) as rank from original_document, to_tsquery('english', 'FRED') query WHERE query @@ coalesce(ts_vector,'') order by rank desc;
+select original_document_id, document_ts_vector, ts_rank(document_ts_vector, query) as rank from original_document, to_tsquery('english', 'FRED') query WHERE query @@ coalesce(document_ts_vector,'') order by rank desc;