Changeset 5524


Ignore:
Timestamp:
27/07/09 16:18:00 (10 years ago)
Author:
sdonegan
Message:

Updated to allow extraction of end date into special column to allow ordering by this

Location:
TI01-discovery/branches/ingestAutomation-upgrade
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

    r5496 r5524  
    293293            self._record.originalFormat + "', '" + self._record.getAuthorsInfo() + "', '" + \ 
    294294            self._record.getParametersInfo() + "', '" + self._record.getScopeInfo() + "', '" + \ 
    295             self._record.dataset_name + "', '" + self._record.datacentre_name + "', '" + self._record.dataset_lastEdit + "', '" + self._record.datasetStartNom+ "');" 
     295            self._record.dataset_name + "', '" + self._record.datacentre_name + "', '" + self._record.dataset_lastEdit + "', '" + self._record.datasetStartNom + "', '" + self._record.datasetEndNom + "');" 
    296296             
    297297        sqlCmd = sqlCmd.replace("'NULL'","NULL") 
     
    325325            self._record.discovery_id + "', '" + self._record.docType + "', '" + \ 
    326326            self._record.originalFormat + "', '" + self._record.getAuthorsInfo() + "', '" + \ 
    327             self._record.getParametersInfo() + "', '" + self._record.getScopeInfo() + "', '" + str(self._record.scn) + "', '" + self._record.dataset_name + "', '" + self._record.datacentre_name + "', '" + self._record.dataset_lastEdit + "', '" + self._record.datasetStartNom+ "');"  
     327            self._record.getParametersInfo() + "', '" + self._record.getScopeInfo() + "', '" + str(self._record.scn) + "', '" + self._record.dataset_name + "', '" + self._record.datacentre_name + "', '" + self._record.dataset_lastEdit + "', '" + self._record.datasetStartNom + "', '" + self._record.datasetEndNom + "');"  
    328328             
    329329        #sort out any NULL values" 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

    r5464 r5524  
    3333         
    3434    #def __init__(self, filename, ndg_dataprovider, datacentre_groups, datacentre_namespace, discovery_id, xq, docType): 
    35     def __init__(self, filename, ndg_dataprovider, datacentre_groups, datacentre_namespace, discovery_id,datasetName,datacentreName,datasetLastEditUpdateDate,datasetStartDateNom, xq, docType): 
     35    def __init__(self, filename, ndg_dataprovider, datacentre_groups, datacentre_namespace, discovery_id,datasetName,datacentreName,datasetLastEditUpdateDate,datasetStartDateNom, datasetEndDateNom, xq, docType): 
    3636                  
    3737        logging.info("Setting up Postgres record for file, " + filename) 
     
    5858        self.datacentre_name = datacentreName 
    5959        self.datasetStartNom = datasetStartDateNom 
     60        self.datasetEndNom = datasetEndDateNom 
    6061 
    6162        self._molesFormat = None    # initialise this, so we can guarantee a value - to avoid using getattr 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/Utilities.py

    r5464 r5524  
    267267            self.metadataCreationDate=helper.getText(et,'DIF_Creation_Date') 
    268268            self.datasetStartDateNom = helper.getText(et,'Temporal_Coverage/Start_Date') 
     269            self.datasetEndDateNom = helper.getText(et,'Temporal_Coverage/Stop_Date') 
    269270             
    270271        elif self._datacentre_format == 'MDIP': 
     
    275276            self.metadataCreationDate=helper.getText(et,'DateOfUpdateOfMetadata') 
    276277            self.datasetStartDateNom = helper.getText(et,'Date/DatasetStartDate') 
     278            self.datasetEndDateNom = helper.getText(et,'Date/DatasetEndDate') 
    277279        else: 
    278280            raise TypeError,'idget does not support datatype [%s]'%dataType 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/abstractdocumentingester.py

    r5523 r5524  
    138138                        datasetLastEditUpdateDate = basicParameters.metadataCreationDate 
    139139                        datasetStartDateNom = basicParameters.datasetStartDateNom 
     140                        datasetEndDateNom = basicParameters.datasetEndDateNom 
    140141                         
    141142                        #record whats attempting to be ingested 
     
    145146                        record = PostgresRecord(filename, self._NDG_dataProvider, \ 
    146147                                                            self._datacentre_groups, self._datacentre_namespace, \ 
    147                                                             discoveryID,datasetName,datacentreName,datasetLastEditUpdateDate,datasetStartDateNom, \ 
     148                                                            discoveryID,datasetName,datacentreName,datasetLastEditUpdateDate,datasetStartDateNom,datasetEndDateNom, \ 
    148149                                                            self._xq, self._datacentre_format) 
    149150                         
  • TI01-discovery/branches/ingestAutomation-upgrade/database/ingest_procedures.sql

    r5463 r5524  
    2121 
    2222DROP FUNCTION create_document(filename_in varchar(255), discovery_id_in varchar(255),  
    23         doc_type_in text, original_document_in text, authors text, parameters text, scope text, dataset_name_in text, datacentre_name_in text, metadata_update_date_in timestamp, dataset_start_date_in timestamp) CASCADE; 
     23        doc_type_in text, original_document_in text, authors text, parameters text, scope text, dataset_name_in text, datacentre_name_in text, metadata_update_date_in timestamp, dataset_start_date_in timestamp, dataset_end_date_in timestamp) CASCADE; 
    2424CREATE FUNCTION create_document(filename_in varchar(255), discovery_id_in varchar(255),  
    25         doc_type_in text, original_document_in text, authors text, parameters text, scope text, dataset_name_in text, datacentre_name_in text, metadata_update_date_in timestamp, dataset_start_date_in timestamp) RETURNS integer AS  
     25        doc_type_in text, original_document_in text, authors text, parameters text, scope text, dataset_name_in text, datacentre_name_in text, metadata_update_date_in timestamp, dataset_start_date_in timestamp, dataset_end_date_in timestamp) RETURNS integer AS  
    2626$$ 
    2727        DECLARE 
     
    3131        INSERT INTO ORIGINAL_DOCUMENT (original_document_id, original_document_filename,  
    3232            discovery_id, original_format, original_document, document_ts_vector, authors_ts_vector,  
    33             parameters_ts_vector, scope_ts_vector, create_date, harvest_count, scn, dataset_name, datacentre_name, dataset_metadata_update_date, dataset_start_date_nom) VALUES ( 
     33            parameters_ts_vector, scope_ts_vector, create_date, harvest_count, scn, dataset_name, datacentre_name, dataset_metadata_update_date, dataset_start_date_nom, dataset_end_date_nom ) VALUES ( 
    3434            DEFAULT, filename_in, discovery_id_in, doc_type_in, original_document_in,  
    3535                        to_tsvector('english', original_document_in), to_tsvector('english', authors),  
    36                         to_tsvector('english', parameters), to_tsvector('english', scope), current_timestamp, 1, 1, dataset_name_in, datacentre_name_in, metadata_update_date_in, dataset_start_date_in); 
     36                        to_tsvector('english', parameters), to_tsvector('english', scope), current_timestamp, 1, 1, dataset_name_in, datacentre_name_in, metadata_update_date_in, dataset_start_date_in, dataset_end_date_in); 
    3737 
    3838                SELECT original_document_id INTO db_id FROM ORIGINAL_DOCUMENT WHERE discovery_id = discovery_id_in; 
  • TI01-discovery/branches/ingestAutomation-upgrade/database/original_document.sql

    r5462 r5524  
    1515                dataset_metadata_update_date timestamp, 
    1616                dataset_start_date_nom timestamp, -- simplest method to allow ordering by date when no date info present in temporal table!! 
     17                dataset_end_date_nom timestamp, -- ditto for end data ordering 
    1718                original_document text, 
    1819                document_ts_vector      tsvector, 
Note: See TracChangeset for help on using the changeset viewer.