Ignore:
Timestamp:
12/11/08 10:38:38 (11 years ago)
Author:
cbyrom
Message:

Move FileUtilities? module to ndgUtils to allow re-use elsewhere + un-oo
this to allow easier use as simple script methods.

Location:
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
Files:
1 deleted
2 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

    r4257 r4423  
    1717import csml.csml2Moles.molesReadWrite as MRW 
    1818from ndgUtils.ndgObject import ndgObject 
    19 from FileUtilities import FileUtilities 
     19import ndgUtils.fileutilities as FileUtilities 
    2020from SpatioTemporalData import SpatioTemporalData 
    2121import keywordAdder 
     
    6161        self._allDocs = []  # array to store all the transformed docs - for easy retrieval by the DAO 
    6262 
    63         self._fileUtils = FileUtilities() 
    64  
    6563        # get the dir of the file - needed by the xquery to use as the target collection 
    6664        tmp = filename.split('/') 
     
    136134        logging.info("Creating moles file on system - for use with other xquery transforms") 
    137135        self._molesDir = self._dir + "/moles/" 
    138         self._fileUtils.setUpDir(self._molesDir) 
     136        FileUtilities.setUpDir(self._molesDir) 
    139137         
    140138        if self._molesFormat is None: 
    141139            self.doMolesTransform() 
    142140             
    143         self._fileUtils.createFile(self._molesDir + self.shortFilename, self._molesFormat) 
     141        FileUtilities.createFile(self._molesDir + self.shortFilename, self._molesFormat) 
    144142        logging.info("Moles file created - at %s" %self._molesDir) 
    145143         
     
    189187        # NB, running directly at the command line leads to problems with the interpretation of $ characters 
    190188        xqFile = "currentQuery.xq" 
    191         self._fileUtils.createFile(xqFile, xquery) 
     189        FileUtilities.createFile(xqFile, xquery) 
    192190 
    193191        # Now do the transform 
     
    249247        tmpDir = os.getcwd() + "/tmp/" 
    250248        tmpKeywordsDir = os.getcwd() + "/keywordsAdded/" 
    251         self._fileUtils.setUpDir(tmpDir) 
    252         self._fileUtils.setUpDir(tmpKeywordsDir) 
     249        FileUtilities.setUpDir(tmpDir) 
     250        FileUtilities.setUpDir(tmpKeywordsDir) 
    253251        tmpFile = 'tmpFile.xml' 
    254         self._fileUtils.createFile(tmpDir + tmpFile, self._molesFormat) 
     252        FileUtilities.createFile(tmpDir + tmpFile, self._molesFormat) 
    255253 
    256254        keywordAdder.main(tmpDir, tmpKeywordsDir, self._datacentre_groups) 
     
    262260         
    263261        # Finally, tidy up temp dirs 
    264         self._fileUtils.cleanDir(tmpDir) 
    265         self._fileUtils.cleanDir(tmpKeywordsDir) 
     262        FileUtilities.cleanDir(tmpDir) 
     263        FileUtilities.cleanDir(tmpKeywordsDir) 
    266264        logging.info("Completed adding keywords") 
    267265         
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

    r4258 r4423  
    99from time import strftime 
    1010from SchemaNameSpace import SchemaNameSpace 
    11 import ndgUtils 
    1211from ndgUtils.ndgXqueries import ndgXqueries 
    13 from FileUtilities import FileUtilities 
     12import ndgUtils.fileutilities as FileUtilities 
    1413from PostgresRecord import PostgresRecord 
    1514from PostgresDAO import PostgresDAO 
     
    205204                        self.usage() 
    206205                 
    207                 # create file utils object to do various file related stuff 
    208                 fileUtils = FileUtilities() 
    209                  
    210206                numfilesproc = 0 
    211207                self._no_files_ingested = 0 
     
    246242                 
    247243                # Create/clear the 'in' directory pristine copy of the discovery records 
    248                 fileUtils.setUpDir(originals_dir) 
     244                FileUtilities.setUpDir(originals_dir) 
    249245                commandline = "find " + self._harvest_home + " -type f -print | xargs -i cp \{\} " + originals_dir 
    250246                logging.info("Executing : " + commandline) 
     
    255251                 
    256252                # Create/clear the directory for the 'out' processed copy of the discovery records. 
    257                 fileUtils.setUpDir(discovery_dir) 
     253                FileUtilities.setUpDir(discovery_dir) 
    258254                     
    259255                #Execute the script which processes/renames the files (changed 08/01/07 to get id from inside file) 
     
    300296                self._xq=ndgXqueries() 
    301297                for libFile in self._xq.xqlib: 
    302                         fileUtils.createFile(libFile, self._xq.xqlib[libFile]) 
     298                        FileUtilities.createFile(libFile, self._xq.xqlib[libFile]) 
    303299                 
    304300                # Process the resulting files and put the data into the postgres DB 
     
    315311                 
    316312                this_backupdir = backupdir_base + "_originals/" 
    317                 #fileUtils.makeBackUp(originals_dir, this_backupdir) 
     313                FileUtilities.makeBackUp(originals_dir, this_backupdir) 
    318314                 
    319315                #Clear out the original harvest records area and discovery dir 
    320                 fileUtils.cleanDir(originals_dir) 
    321                 #fileUtils.cleanDir(discovery_dir) 
     316                FileUtilities.cleanDir(originals_dir) 
     317                FileUtilities.cleanDir(discovery_dir) 
    322318                 
    323319                logging.info("oai_document_ingest processing complete:") 
    324320                if self._no_problem_files == 0: 
    325321                        logging.info("All files successfully processed - cleaning harvest directory") 
    326                         #fileUtils.cleanDir(self._harvest_home) 
     322                        FileUtilities.cleanDir(self._harvest_home) 
    327323                else: 
    328324                        logging.error("Problems experienced with %s files" %self._no_problem_files) 
Note: See TracChangeset for help on using the changeset viewer.