Changeset 3810 for TI01-discovery


Ignore:
Timestamp:
21/04/08 13:10:30 (11 years ago)
Author:
cbyrom
Message:

Create Logger class to standardise output - and allow this to be
switched on/off with a verbose flag.
Implement the logger in several other classes + extend available logging in these.
Also, add a 'usage' method to oai_ingest_new to show proper usage of script.

Location:
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed

  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/FileUtilities.py

    r3800 r3810  
    1 import os 
    2 import sys 
     1import os, sys 
     2from Logger import Logger 
     3 
    34class FileUtilities: 
    4         ''' 
     5    ''' 
    56        Various helper methods for setting up and cleaning filesystems  
    6         C Byrom Apr 08 
    7         ''' 
    8  
    9 def makepath(path): 
    10     """ creates missing directories for the given path and 
    11         returns a normalized absolute version of the path. 
    12  
    13     - if the given path already exists in the filesystem 
    14       the filesystem is not modified. 
    15  
    16     - otherwise makepath creates directories along the given path 
    17       using the dirname() of the path. You may append 
    18       a '/' to the path if you want it to be a directory path. 
    19  
    20     from holger@trillke.net 2002/03/18 
    21     """ 
    22     from os import makedirs 
    23     from os.path import normpath,dirname,exists,abspath 
    24  
    25     print "INFO: Creating dir: %s" %path  
    26     dpath = normpath(dirname(path)) 
    27     if not exists(dpath): 
    28         makedirs(dpath) 
    29     return normpath(abspath(path)) 
    30  
    31 def setUpDir(dir): 
    32         ''' 
    33         Clean out specified directory - or create this, if it doesn't already exist 
    34         @param dir: directory to set up/clean 
    35         ''' 
    36         print "Setting up directory, %s" %dir 
    37         if os.path.isdir(dir): 
    38                 cleanDir(dir) 
    39         else: 
    40                 try: 
    41                         makepath(dir) 
    42                 except: 
    43                         sys.exit("Failed at setting up directory, %s" %dir) 
     7    C Byrom Apr 08 
     8    ''' 
     9    logger = None 
     10    def __init__(self, inLogger): 
     11        ''' 
     12        Constructor - to set up the utils with a default logger 
     13        @param logger: logger to use; if none specified, use default logger.  NB, logger must  
     14                support a 'printOutput' method 
     15        ''' 
     16        if inLogger is not None: 
     17            self.logger = inLogger 
     18        else: 
     19            self.logger = Logger(True) 
    4420 
    4521 
    46 def cleanDir(dir): 
    47         ''' 
    48         Remove all files from the specified directory 
    49         @param dir: directory to clean  
    50         ''' 
    51         print "INFO: Cleaning dir %s" %dir 
    52         commandline = "ls -1 " + dir + " | xargs -i rm " + dir + "{\}" 
    53         print "INFO: Executing : " + commandline 
    54         status = os.system(commandline) 
    55         if status !=0: 
    56             sys.exit("Failed at cleaning out directory, %s" %dir) 
     22    def makepath(self, path): 
     23        ''' creates missing directories for the given path and 
     24            returns a normalized absolute version of the path. 
     25     
     26        - if the given path already exists in the filesystem 
     27          the filesystem is not modified. 
     28     
     29        - otherwise makepath creates directories along the given path 
     30          using the dirname() of the path. You may append 
     31          a '/' to the path if you want it to be a directory path. 
     32     
     33        from holger@trillke.net 2002/03/18 
     34        ''' 
     35        from os import makedirs 
     36        from os.path import normpath,dirname,exists,abspath 
     37     
     38        self.logger.printOutput("INFO: Creating dir: " + path)  
     39        dpath = normpath(dirname(path)) 
     40        if not exists(dpath): 
     41            makedirs(dpath) 
     42        return normpath(abspath(path)) 
    5743 
    58          
    59 def makeBackUp(original_dir, backup_dir): 
    60         ''' 
    61         Copy contents of original dir into backup dir 
    62         @param original_dir: dir to backup 
    63         @param backup_dir: dir to backup to   
    64         ''' 
    65         print "Creating backup directory (%s --> %s)" %original_dir %backup_dir 
    66         commandline = "mkdir " + backup_dir 
    67         print "INFO: Executing : " + commandline 
    68         status = os.system(commandline) 
    69         if status !=0: 
    70                 sys.exit("Failed at creating backup directory %s" %backup_dir) 
     44     
     45    def setUpDir(self, dir): 
     46        ''' 
     47        Clean out specified directory - or create this, if it doesn't already exist 
     48        @param dir: directory to set up/clean 
     49        ''' 
     50        self.logger.printOutput("Setting up directory, " + dir) 
     51        if os.path.isdir(dir): 
     52            self.cleanDir(dir) 
     53        else: 
     54            try: 
     55                makepath(dir) 
     56            except: 
     57                sys.exit("Failed at setting up directory, %s" %dir) 
    7158 
    72                 commandline = "ls -1 " + original_dir + " | xargs -i cp " + original_dir + "{\} " + backup_dir 
    73                 print "INFO: Executing : " + commandline 
    74                 status = os.system(commandline) 
    75                 if status !=0: 
    76                         sys.exit("Failed at copying to backup directory %s" %backup_dir) 
    77          
     59 
     60    def cleanDir(self, dir): 
     61        ''' 
     62        Remove all files from the specified directory 
     63        @param dir: directory to clean  
     64        ''' 
     65        self.logger.printOutput("INFO: Cleaning dir " + dir) 
     66        commandline = "ls -1 " + dir + " | xargs -i rm " + dir + "{\}" 
     67        self.logger.printOutput("INFO: Executing : " + commandline) 
     68         
     69        status = os.system(commandline) 
     70        if status !=0: 
     71            sys.exit("Failed at cleaning out directory, %s" %dir) 
     72 
     73         
     74    def makeBackUp(self, original_dir, backup_dir): 
     75        ''' 
     76        Copy contents of original dir into backup dir 
     77        @param original_dir: dir to backup 
     78        @param backup_dir: dir to backup to   
     79        ''' 
     80        self.logger.printOutput("Creating backup directory (" + original_dir + \ 
     81                                " --> " + backup_dir + ")") 
     82        commandline = "mkdir " + backup_dir 
     83        self.logger.printOutput("INFO: Executing : " + commandline) 
     84        status = os.system(commandline) 
     85        if status !=0: 
     86            sys.exit("Failed at creating backup directory %s" %backup_dir) 
     87     
     88            commandline = "ls -1 " + original_dir + " | xargs -i cp " + original_dir + "{\} " + backup_dir 
     89            self.logger.printOutput("INFO: Executing : " + commandline) 
     90            status = os.system(commandline) 
     91            if status !=0: 
     92                sys.exit("Failed at copying to backup directory %s" %backup_dir) 
     93     

  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/SchemaNameSpace.py

    r3797 r3810  
    11import sys 
     2from Logger import Logger 
    23 
    34class SchemaNameSpace: 
    … …  
    67    NB, only currently handles correction of DIF files 
    78    ''' 
    8     def __init__(self,infile,outfile,format): 
     9    def __init__(self,infile,outfile,format, logger): 
    910        ''' 
    1011        Constructor - with the logic to do the namespace change 
    … …  
    1920        for self.line in self.lines: 
    2021            if self.format== "DIF" and self.line.startswith('<DIF'): 
    21                 print "INFO: changing line for %s. output to %s" %(infile,outfile) 
     22                message = "INFO: changing line for %s. output to %s" %(infile,outfile) 
     23                if (logger is None): 
     24                    print message 
     25                else: 
     26                    logger.printOutput(message) 
    2227                self.line='<DIF xmlns="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n' 
    2328            self.ww.write(self.line) 
    2429        self.ff.close() 
    2530        self.ww.close() 
    26  
    27 if __name__=="__main__": 
    28     import sys 
    29     f=sys.argv[1] 
    30     w=sys.argv[2] 
    31     form=sys.argv[3] 
    32     SchemaNameSpace(f,w,form) 

  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_ingest_new.py

    r3800 r3810  
    2727# + remove dependency on eXist DB 
    2828 
    29 import os 
    30 import sys 
    31 import commands 
    32 import string 
     29import os, sys, commands, string, getopt 
    3330import keywordAdder 
    3431from SchemaNameSpace import SchemaNameSpace 
    … …  
    4138from ndgUtils.ndgDirectory import ndgDirectory 
    4239from ndgUtils.eXistInterface import ndg_eXist 
    43 from FileUtilities import setUpDir, cleanDir, makeBackUp 
     40from FileUtilities import FileUtilities 
    4441from PostgresRecord import PostgresRecord 
    45 import PostgresDAL 
    46  
     42import PostgresDAO 
     43import renderEntity 
     44from Logger import Logger 
     45 
     46         
    4747def getID(filename): 
    4848        '''  
    … …  
    5252        @return: ID - id to use to refer to the document 
    5353        ''' 
     54        logger.printOutput("INFO: Retrieving identifier for metadata record " + filename) 
    5455        xml=file(filename).read() 
    5556        if datacentre_format == "DIF": 
    … …  
    6162        else: 
    6263            sys.exit("Only handles DIF or MDIP here.") 
     64             
     65        logger.printOutput("Found identifier: " + ID) 
    6366        return ID 
     67 
    6468 
    6569def addFileToPostgresDB(filename): 
    … …  
    6872        data in the process 
    6973        ''' 
    70         print "Adding file, %s, to postgres DB" %filename 
     74        logger.printOutput("Adding file, " + filename + ", to postgres DB") 
    7175         
    7276        # first of all create a PostgresRecord - this object represents all the data required 
    7377        # for a DB entry 
    74         record = PostgresRecord(filename, NDG_dataProvider) 
    75         dal = PostgresDAL(record) 
    76         dal.createOrUpdateRecord() 
     78        record = PostgresRecord(filename, NDG_dataProvider, targetCollection, datacentre_namespace, 'discovery_idTEST', xq, datacentre_format) 
     79        dao = PostgresDAO(record) 
     80        dao.createOrUpdateRecord() 
     81 
    7782 
    7883def getConfigDetails(datacentre): 
    … …  
    9095        global datacentre_config_filename, NDG_dataProvider 
    9196        datacentre_config_filename = base_dir + datacentre + "_config.properties" 
    92         print "INFO: Retrieving data from datacentre config file = %s" %datacentre_config_filename 
     97        logger.printOutput("INFO: Retrieving data from datacentre config file, " + datacentre_config_filename) 
    9398         
    9499        # Check this file exists; if not, assume an invalid datacentre has been specified 
    … …  
    119124            sys.exit("Failed at getting harvested records directory stage. datacentre config file tried = %s" %datacentre_config_filename) 
    120125         
    121         print "INFO: harvested records are in %s" %harvest_home 
     126        logger.printOutput("INFO: harvested records are in " + harvest_home) 
    122127         
    123128        if datacentre_groups == "": 
    124             print "INFO: No groups/keywords set for datacentre %s" %datacentre 
     129            logger.printOutput("INFO: No groups/keywords set for datacentre " + datacentre) 
    125130        else: 
    126             print "INFO: datacentre groups/keywords = %s" %datacentre_groups 
     131            logger.printOutput("INFO: datacentre groups/keywords: " + datacentre_groups) 
    127132         
    128133        if datacentre_format == "": 
    129134            sys.exit("Failed at stage: getting datacentre format. datacentre config file tried = %s" %datacentre_config_filename) 
    130135         
    131         print "INFO: format being harvested = %s" %datacentre_format 
     136        logger.printOutput("INFO: format being harvested: " + datacentre_format) 
    132137         
    133138        if datacentre_namespace == "": 
    134139            sys.exit("Failed at stage: getting datacentre namespace. datacentre config file tried = %s" %datacentre_config_filename) 
    135140         
    136         print "INFO: datacentre namespace = %s" %datacentre_namespace 
     141        logger.printOutput("INFO: datacentre namespace: " + datacentre_namespace) 
    137142        print lineSeparator 
    138          
    139  
    140 def usage(): 
    141         ''' 
    142         Display input params for the script 
    143         ''' 
    144         print "Usage: python oai_ingest.py <datacentre> (<dbInfoFile>)" 
    145         print " - where:\n   <datacentre> is the data centre to ingest data from; and" 
    146         print "   <dbInfoFile> provides info on the eXist DB to use to do XQuery transforms" 
    147         sys.exit() 
    148143 
    149144 
    … …  
    153148        into the other required doc types 
    154149        ''' 
    155         print "INFO: Putting original docs in eXist..." 
     150        logger.printOutput("INFO: Putting original docs in eXist...") 
    156151        # this command creates the targetCollection in eXist from the contents of the discovery_corrected_dir 
    157152        commandline = "$EXIST_HOME/bin/client.sh -c " + targetCollection + " -u admin -P " + \ 
    158153                db_admin + " -p " + discovery_corrected_dir 
    159         print lineSeparator 
    160         print "INFO: Executing : actual command to ingest into exist db", commandline 
     154        logger.printOutput(lineSeparator) 
     155        logger.printOutput("INFO: Executing : actual command to ingest into exist db: " + commandline) 
    161156        status = os.system(commandline) 
    162157        if status !=0: 
    163158            sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status)) 
    164         print "INFO: Original docs now in eXist" 
    165         print lineSeparator 
     159        logger.printOutput("INFO: Original docs now in eXist") 
     160        logger.printOutput(lineSeparator) 
    166161 
    167162 
    … …  
    172167        ''' 
    173168        # First get the list of discovery record ids from the db collection 
    174         print "INFO: Creating minimoles documents from original docs in eXist" 
    175         if datacentre_format == 'DIF': 
    176             ndgDir=ndgDirectory(targetCollection,host, datacentre_format.upper()) 
    177         else: 
    178             print 'ERROR: mini-moles creation does not handle MDIP yet! So this WILL FAIL (probably)' 
     169        logger.printOutput("INFO: Creating minimoles documents from original docs in eXist") 
     170        #       if datacentre_format == 'DIF': 
     171     
     172        ndgDir = ndgDirectory(targetCollection, host, datacentre_format.upper()) 
     173    #   else: 
     174    #       print 'ERROR: mini-moles creation does not handle MDIP yet! So this WILL FAIL (probably)' 
    179175         
    180176        #create the mini-moles for each Discovery record in the collection 
    … …  
    186182            # now create the xquery 
    187183            # sort out the output ID stuff ... 
     184            xQueryType = 'dif2moles' 
    188185            if NDG_dataProvider: 
    189186                discObj=ndgObject(disc_id) 
    190                 xquery=xq.actual('dif2moles',targetCollection,discObj.repository,discObj.localID) 
     187                xquery=xq.actual(xQueryType, targetCollection, discObj.repository, discObj.localID) 
     188                print "REPOSITORY: %s, DISC_ID: %s" %(discObj.repository, discObj.localID) 
    191189            else: 
    192                 xquery=xq.actual('dif2moles',targetCollection,datacentre_namespace,disc_id) 
     190                xquery=xq.actual(xQueryType, targetCollection, datacentre_namespace, disc_id) 
    193191 
    194192            # and then sort out the input ID stuff 
    195             xquery=xquery.replace('Input_Entry_ID',disc_id) 
     193            xquery=xquery.replace('Input_Entry_ID', disc_id) 
    196194            xquery=xquery.replace('repository_localid', datacentre_namespace ) 
    197195 
    … …  
    218216                addKeywords() 
    219217                 
    220                 # ingest the created discovery minimum molesrecords into eXist db. 
    221                 commandline = "$EXIST_HOME/bin/client.sh -c /db/discovery/moles -u admin -P " + db_admin + \ 
    222                 " -p " + finalmoles_dir 
    223                 print "INFO: Executing : actual command to ingest into exist db." 
    224                 status = os.system(commandline) 
    225                 if status !=0: 
    226                     sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status)) 
     218        # ingest the created discovery minimum molesrecords into eXist db. 
     219        commandline = "$EXIST_HOME/bin/client.sh -c " + miniMolesCollection + " -u admin -P " + db_admin + \ 
     220        " -p " + finalmoles_dir 
     221        print "INFO: Executing : actual command to ingest into exist db." 
     222        status = os.system(commandline) 
     223        if status !=0: 
     224            sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status)) 
    227225 
    228226 
    … …  
    234232                " -u admin -P " + db_admin + " -m " 
    235233        collections = ['/db/xqueryLib', '/db/discovery/moles'] 
    236         print lineSeparator 
     234        logger.printOutput(lineSeparator) 
    237235        for collection in collections: 
    238236                cmd = commandline + collection + " -p" 
    239                 print "INFO: Executing : actual command to create DB collection ", cmd 
     237                logger.printOutput("INFO: Executing : actual command to create DB collection: " +  cmd) 
    240238                status = os.system(cmd) 
    241239                print status 
    … …  
    243241                        sys.exit("Failed to create DB collection. Status = %s" %(status)) 
    244242         
    245  
    246243         
    247244def addKeywords(): 
    … …  
    249246            commandline = "ls -1 " + minimoles_dir + " | xargs -i mv " + minimoles_dir + \ 
    250247                "{\} " + finalmoles_dir 
    251             print "INFO: Executing : " + commandline 
     248            logger.printOutput("INFO: Executing : " + commandline) 
    252249            status = os.system(commandline) 
    253250            if status !=0: 
    … …  
    264261        commandline = "$EXIST_HOME/bin/client.sh -c /db " + \ 
    265262                " -u admin -P " + db_admin + " -p " + base_dir + 'xquery' 
    266         print lineSeparator 
    267         print "INFO: Executing : actual command to ingest into exist db", commandline 
     263        logger.printOutput(lineSeparator) 
     264        logger.printOutput("INFO: Executing : actual command to ingest into exist db: " + commandline) 
    268265        status = os.system(commandline) 
    269         print status 
    270266        if status !=0: 
    271267            sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status)) 
    272268         
    273269 
    274          
     270def usage(): 
     271        ''' 
     272        Display input params for the script 
     273        ''' 
     274        print "Usage: python -v oai_ingest.py <datacentre> (<dbInfoFile>)" 
     275        print " - where:\n   <datacentre> is the data centre to ingest data from; and" 
     276        print "   <dbInfoFile> provides info on the eXist DB to use to do XQuery transforms" 
     277        print " -v - verbose mode for output logging" 
     278        sys.exit(2) 
     279         
     280 
    275281lineSeparator = "-----------------------------" 
    276282print lineSeparator 
    … …  
    278284print lineSeparator 
    279285 
    280 if (len(sys.argv) < 2 or len(sys.argv) > 3): 
     286verboseMode = False 
     287 
     288# check for verbose option 
     289try: 
     290    opts, args = getopt.getopt(sys.argv[1:], "v") 
     291except getopt.GetoptError, err: 
     292    # print help information and exit: 
     293    print str(err) # will print something like "option -a not recognized" 
     294    usage() 
     295     
     296for o, a in opts: 
     297    if o == "-v": 
     298        verboseMode = True 
     299 
     300if (len(args) < 1 or len(args) > 2): 
    281301        usage() 
    282302else: 
    283     datacentre = sys.argv[1] 
     303    datacentre = args[0] 
    284304 
    285305# set the default password file 
    286306dbinfoname = "ingest.txt" 
    287 if (len(sys.argv) == 3): 
    288         dbinfoname = sys.argv[2] 
     307if (len(args) == 2): 
     308        dbinfoname = args[1] 
     309 
     310# set up the logger to use 
     311logger = Logger(verboseMode) 
     312 
     313# set up the file utils to use this logger 
     314fileUtils = FileUtilities(logger) 
    289315 
    290316status = 0 
    291317numfilesproc = 0 
    292 base_dir = "/usr/local/WSClients/OAIBatch/" # this is the base dir that the script is ran from 
    293 #base_dir = os.getcwd() + "/"# this is the base dir that the script is ran from 
     318#base_dir = "/usr/local/WSClients/OAIBatch/" # this is the base dir that the script is ran from 
     319base_dir = os.getcwd() + "/"# this is the base dir that the script is ran from 
    294320         
    295321data_dir = base_dir + "data/" + datacentre  # dir relating to the specified datacentre docs 
    … …  
    300326# Other settings and constants 
    301327date_string = commands.getoutput("date +'%y%m%d_%H%M'") 
     328  
    302329#os.putenv('EXIST_HOME', '/usr/local/exist-client') 
    303330os.putenv('EXIST_HOME', '/home/users/cbyrom/opt/eXist') 
    … …  
    314341#any records to harvest? 
    315342if len( os.listdir(harvest_home)) == 0: 
    316     print "INFO: Nothing to harvest this time from %s" %datacentre 
     343    logger.printOutput("INFO: Nothing to harvest this time from " + datacentre) 
    317344    sys.exit() 
    318345 
    … …  
    342369 
    343370# Create/clear the 'in' directory pristine copy of the discovery records 
    344 setUpDir(originals_dir) 
     371fileUtils.setUpDir(originals_dir) 
    345372commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} " + originals_dir 
    346 print "INFO: Executing : " + commandline 
     373logger.printOutput("INFO: Executing : " + commandline) 
    347374status = os.system(commandline) 
     375 
    348376if status !=0: 
    349377    sys.exit("Failed at making pristine copy stage") 
    350378 
    351379# Create/clear the directory for the 'out' processed copy of the discovery records. 
    352 setUpDir(discovery_dir) 
     380fileUtils.setUpDir(discovery_dir) 
    353381     
    354382# Create/clear the directory for the 'out' namespace corrected copy of the discovery records. 
    355 setUpDir(discovery_corrected_dir) 
     383fileUtils.setUpDir(discovery_corrected_dir) 
    356384 
    357385# The file config.properties contains the location of the particular datacentres harvested records. 
    358386# Copy the datacentre specific version of config to config.properties file. 
    359387commandline = "cp " + datacentre_config_filename + " " + base_dir + "config.properties" 
    360 print "INFO: Executing : " + commandline 
     388logger.printOutput("INFO: Executing : " + commandline) 
    361389status = os.system(commandline) 
    362390if status !=0: 
    … …  
    365393#Execute the script which processes/renames the files (changed 08/01/07 to get id from inside file) 
    366394# NB, this copies files from the original dir to the discovery dir 
    367 print lineSeparator 
    368 print "INFO: Renaming files:" 
     395logger.printOutput(lineSeparator) 
     396logger.printOutput("INFO: Renaming files:") 
    369397for filename in os.listdir(originals_dir): 
    370398        if filename.find('.xml') != -1: 
    371399                original_filename = originals_dir + filename 
    372400                ident=getID(original_filename) 
    373                 print "INFO: ID extracted from the discovery record = %s" %ident 
    374401                if NDG_dataProvider: 
    375402                    new_filename = discovery_dir + ident.replace(":","__")+".xml" 
    … …  
    379406                    new_filename = discovery_dir + "/" +datacentre_namespace+ "__"+datacentre_format+ "__"+ ident +".xml" 
    380407  
    381                 print "original file = %s\nnewfile = %s" %(original_filename, new_filename) 
     408                    logger.printOutput("original file = " + original_filename) 
     409                    logger.printOutput("newfile = " + new_filename) 
    382410                 
    383411                try: 
    … …  
    391419#replace any namespace declarations with a standard one which we know works in NDG 
    392420# NB, this copies files from the discovery dir to the discovery corrected dir 
    393 print lineSeparator 
    394 print "INFO: Correcting namespaces of files (corrected files in %s)" %discovery_corrected_dir 
     421logger.printOutput(lineSeparator) 
     422logger.printOutput("INFO: Correcting namespaces of files (corrected files in "  + \ 
     423                    discovery_corrected_dir + ")") 
    395424for filename in os.listdir(discovery_dir): 
    396425        if filename.find('.xml') != -1: 
    … …  
    398427                    corrected_filename = discovery_corrected_dir + filename 
    399428                    try: 
    400                        SchemaNameSpace(in_filename, corrected_filename,datacentre_format) 
     429                       SchemaNameSpace(in_filename, corrected_filename,datacentre_format, logger) 
    401430                    except: 
    402431                       print "ERROR: SchemaNameSpace failed on file %s"%in_filename 
    … …  
    408437# now set up the required XQueries and check xqueries libs are available from eXist 
    409438targetCollection = "/db/discovery/original/" +datacentre_format+ "/" +datacentre_namespace 
     439miniMolesCollection = "/db/discovery/moles" 
    410440xq=ndgXqueries() 
    411441xmldb=ndg_eXist(db='' + host + '') 
    … …  
    418448minimoles_dir = base_dir + 'MINIMOLES/' 
    419449finalmoles_dir = base_dir + 'FINALMOLES/' 
    420 setUpDir(minimoles_dir) 
    421 setUpDir(finalmoles_dir) 
     450fileUtils.setUpDir(minimoles_dir) 
     451fileUtils.setUpDir(finalmoles_dir) 
    422452createEXistMiniMoles() 
    423453 
    424454# TODO: need to skip the eXist steps below and just populate the postGres DB 
    425 # 1. using the discovery_corrected_dir - (do we need the non corrected files?) - import into original column 
    426455# 2. use the correct xquery to transform the original doc directly and store t 
    427456 
    … …  
    432461        addFileToPostgresDB(discovery_corrected_dir + filename) 
    433462 
    434 #SpaceTimeIngestFromMOLES.main("./FINALMOLES") 
    435  
    436 #Extract the spatiotemporal info and put into NEW postgis tables 
    437 #SpaceTimeIngestPostgisFromMOLES.main("./FINALMOLES") 
    438  
    439463#Make copies of discovery and oai/originals and DIF2MOLES areas to backup area for tape backups 
    440464backupdir_base = backupdir + datacentre + "_" + date_string 
    441465this_backupdir = backupdir_base + "_originals" 
    442 makeBackUp(originals_dir, this_backupdir) 
     466fileUtils.makeBackUp(originals_dir, this_backupdir) 
    443467 
    444468this_backupdir = backupdir_base + "_discovery" 
    445 makeBackUp(discovery_dir, this_backupdir) 
     469fileUtils.makeBackUp(discovery_dir, this_backupdir) 
    446470 
    447471this_backupdir = backupdir_base + "_FINALMOLES" 
    448 makeBackUp("./FINALMOLES", this_backupdir) 
     472fileUtils.makeBackUp("./FINALMOLES", this_backupdir) 
    449473 
    450474#Clear out the original harvest records area and FINALMOLES 
    451 cleanDir("./FINALMOLES") 
    452 cleanDir(harvest_home) 
     475fileUtils.cleanDir("./FINALMOLES") 
     476fileUtils.cleanDir(harvest_home) 
    453477 
    454478print "======================================================" 
Note: See TracChangeset for help on using the changeset viewer.