source: TI03-DataExtractor/trunk/setup.py @ 794

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/setup.py@794
Revision 794, 16.3 KB checked in by astephen, 13 years ago (diff)

Unstable but latest version with multi-variable support and split hooks
for CDML and CSML.

Line 
1#!/usr/bin/env python
2
3"""
4setup.py
5========
6
7Set up script for the Data Extractor package.
8
9The Data Extractor is a server package (dxs) that is exposed
10as a Web Service using python's ZSI SOAP Library.
11
12There is also a client package to primarily support the CGI client
13package (dxc).
14
15This installation includes both server and client and assumes you want
16to install both on your local machine. If you only wish to install either
17server or client use one of the following options:
18
19--dxc-only # only install the client package
20--dxs-only # only install the server package
21
22
23Usage:
24======
25
26    [<your_python>/bin/python] setup.py [options] 
27
28[options] can include the above as well as the usual
29
30build | install
31
32Note that using "--overwrite" will automatically overwrite
33existing directories regardless of whether
34
35"""
36
37from distutils.core import setup
38import sys, os, shutil, time, re
39from ConfigParser import ConfigParser
40version="0.2.1"
41overwrite="OFF"
42
43
44def createSourceBundle(packages=["dxs", "dxc"]):
45    """
46    Creates the packages required and includes the directories needed.
47    """
48    basePackage=packages[0][:2]
49    packMap={"s":"Server", "c":"Client"}
50       
51    for package in packages: 
52        if os.path.isfile("MANIFEST"): os.remove("MANIFEST")
53        pypackage="py"+package
54        packageName="Python Data Extractor %s" % packMap[pypackage[-1]]
55        print "\nBundling up source of python package: %s" % packageName
56   
57        setup(name=pypackage,
58            version=version,
59            description=packageName,
60            author="Ag Stephens",           
61            author_email="a.stephens@rl.ac.uk",
62            url="http://ndg.nerc.ac.uk/DataExtractor",
63            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
64This software may be distributed under the terms of the
65Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
66            packages=[pypackage])   
67           
68        print "Bundling up the source of the non-python stuff and python scripts for: %s" % packageName
69        tarfile="%s-%s.tar" % (package, version)
70        gzipfile=tarfile+".gz"
71        os.system("tar -cvf %s %s" % (tarfile, package))
72        os.system("gzip %s" % tarfile)
73       
74        if not os.path.isfile(gzipfile):
75            raise "Did not successfully package up non-python source for '%s'." % package
76           
77        print "Moving gzip file into distribution..."
78        os.rename(gzipfile, os.path.join("dist", gzipfile))
79
80        if package=="dxc":
81            print "Copying cgi directory..."
82            if os.path.isdir("dist/cgi"):
83                if os.path.isfile("dist/cgi/dxui"):
84                    os.remove("dist/cgi/dxui")
85                os.rmdir("dist/cgi")
86            shutil.copytree("cgi", "dist/cgi")
87           
88            print "Copying web_extras directory..."
89            if os.path.isdir("dist/web_extras"):
90                os.system("rm -fR dist/web_extras")
91            shutil.copytree("web_extras", "dist/web_extras")       
92           
93    print "Copying README.txt and install.conf files..."
94    shutil.copy("README.txt", "dist")
95    shutil.copy("install.sh", "dist")   
96    shutil.copy("install.conf", "dist") 
97    shutil.copy("install.conf", "dist/install.conf.orig")
98       
99    print "Tarring up everything in dist directory..."
100    tarAllFile="%s-%s-all.tar" % (basePackage, version)
101    os.system("cd dist ; tar -cvf %s *" % tarAllFile)
102       
103    if not os.path.isfile(os.path.join("dist", tarAllFile)):
104        raise "Did not successfully package up tar-all file '%s'." % tarAllFile
105       
106
107
108def setupPackages(packages=["dxs", "dxc"]):
109    """
110    Controls the entire setup and creates directories and
111    permissions where necessary to get the ancillary stuff working.
112    """   
113    testImportExternalPackages()
114   
115    packMap={"s":"Server", "c":"Client"}
116       
117    for package in packages: 
118        pypackage="py"+package
119        packageName="Python Data Extractor %s" % packMap[package[-1]]
120           
121        print "Installing the non-python stuff and python scripts for: %s" % packageName
122        unpackageDir="%s-%s" % (package, version)
123        tarfile="%s.tar" % unpackageDir
124        gzipfile=tarfile+".gz"
125        os.system("cd .. ; gunzip %s" % tarfile)       
126        os.system("cd .. ; tar -xvf %s" % tarfile)
127
128        packageDir=os.path.join("..", package)
129        if not os.path.isdir(packageDir):
130            raise "Did not successfully unpack the source files for %s." % package
131           
132        print "Reading configuration file to get %s local details." % package
133        config=ConfigParser()
134        config.read("../install.conf")
135 
136        print "Set the configuration variables that are used to generate others..."
137        for item in ["BASEDIR", "STATIC_WEBSERVER_LOCAL_PATH", "STATIC_WEBSERVER_URL_PATH",
138                     "WEB_EXTRAS_URL_PATH"]:
139            if item.lower() in config.options(package):
140                exec("%s=%s" % (item.upper(), config.get(package, item.lower())))
141                print "SET:", item
142 
143        print "\n\n"
144        configVarDict={}
145        for configVar in config.options(package):
146                value=config.get(package, configVar)
147                print "Setting: %s=%s" % (configVar.upper(), value)
148                exec("%s=%s" % (configVar.upper(), value))
149                configVarDict[configVar.upper()]=value
150
151        print "Now we have parsed the config file we can start copying and making the source directories."
152        print "Installing base package '%s' in: %s" % (package, BASEDIR)
153
154        if overwrite!="ON":
155            checkBeforeMakingDir(BASEDIR)
156           
157            """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % BASEDIR)
158            if destroy!="yes":
159                print "Exiting setup without completing."
160                sys.exit()
161           
162        if not os.path.isdir(BASEDIR):
163            try:
164                os.makedirs(BASEDIR)
165            except:
166                pass"""
167       
168        for item in os.listdir(packageDir):
169            sourceCopy=os.path.join(packageDir, item)
170            target=os.path.join(BASEDIR, item)
171            if os.path.exists(target): 
172                os.system("rm -fr %s" % target)
173           
174            if os.path.isfile(sourceCopy):
175                shutil.copy(sourceCopy, target)
176            elif os.path.isdir(sourceCopy):
177                #if os.path.split(sourceCopy)[-1]=="logs":
178                #    print "Not overwriting logs directory."
179                #    if not os.path.isdir(
180                #else:
181                    shutil.copytree(sourceCopy, target)
182       
183        print "Installed non-python package and scripts under:", BASEDIR
184
185        if "OUTPUT_DIR" in configVarDict.keys():
186            if overwrite!="ON":
187                checkBeforeMakingDir(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP)           
188               
189                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % OUTPUT_DIR)
190                if destroy!="yes":
191                    print "Exiting setup without completing."
192                    sys.exit()   
193            try:
194                os.system("rm -fR %s" % OUTPUT_DIR)
195                os.makedirs(OUTPUT_DIR, OUTPUT_DIR_PERMISSION)
196                #os.chmod(OUTPUT_DIR, OUTPUT_DIR_PERMISSION)
197                os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
198            except:
199                raise "Cannot create directory: %s" % OUTPUT_DIR"""
200
201        for dirToMake in ["OUTPUT_DIR_LOCAL_PATH"]:
202         
203          if dirToMake in configVarDict.keys():
204            dirToMake=eval(configVarDict[dirToMake])     
205            if overwrite!="ON":
206                checkBeforeMakingDir(OUTPUT_DIR_LOCAL_PATH)             
207                   
208           
209                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % dirToMake)
210                if destroy!="yes":
211                    print "Exiting setup without completing."
212                    sys.exit()   
213            try:
214                os.system("rm -fR %s" % dirToMake)
215                os.makedirs(dirToMake, 0755)
216                #os.chmod(dirToMake, 0755)
217                #os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
218            except:
219                raise "Cannot create directory: %s" % dirToMake"""
220               
221        if "CGI_SCRIPT_LOCAL_PATH" in configVarDict.keys():
222            """if overwrite!="ON":
223                destroy=raw_input("Type 'yes' to remove existing script before re-installing: '%s' :" % CGI_SCRIPT_URL_PATH)
224                if destroy!="yes":
225                    print "Exiting setup without completing."
226                    sys.exit()"""   
227            try:
228                shutil.copy("../cgi/dxui", CGI_SCRIPT_LOCAL_PATH)
229                os.chmod(CGI_SCRIPT_LOCAL_PATH, 0755)
230                thisPython=sys.modules["os"].__file__.split("lib")[0]+"bin/python"
231                print thisPython
232                os.system("perl -p -i -w -e 's:<YOUR_PYTHON_HERE>:%s:g;' %s*" % (thisPython, CGI_SCRIPT_LOCAL_PATH))
233                #os.system("perl -p -i -w -e 's!BASEDIR=\".*?\"!BASEDIR=\"%s\"!g;' %s*" % ("/home/as56/abstractWebService/test_installs/20060328/pydxc-0.1/pydxc", CGI_SCRIPT_LOCAL_PATH))
234                print "\nCopied CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
235            except:
236                raise "Cannot move CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
237
238        if "WEB_EXTRAS_LOCAL_PATH" in configVarDict.keys():
239            if overwrite!="ON":
240                checkBeforeMakingDir(WEB_EXTRAS_LOCAL_PATH, 0755)                   
241                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % WEB_EXTRAS_LOCAL_PATH)
242                if destroy!="yes":
243                    print "Exiting setup without completing."
244                    sys.exit()   
245            try:
246                os.system("rm -fR %s" % WEB_EXTRAS_LOCAL_PATH)
247                os.makedirs(WEB_EXTRAS_LOCAL_PATH, 0755)
248                #os.chmod(OUTPUT_DIR, OUTPUT_DIR_PERMISSION)
249                #os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
250            except:
251                raise "Cannot create directory: %s" % WEB_EXTRAS_LOCAL_PATH  """               
252
253            webExtrasDir="../web_extras"
254            for item in os.listdir(webExtrasDir):
255                sourceCopy=os.path.join(webExtrasDir, item)
256                target=os.path.join(WEB_EXTRAS_LOCAL_PATH, item)
257                if os.path.exists(target): 
258                    os.system("rm -fR %s" % target)
259           
260                if os.path.isfile(sourceCopy):
261                    shutil.copy(sourceCopy, target)
262                elif os.path.isdir(sourceCopy):
263                    shutil.copytree(sourceCopy, target)
264       
265            print "Installed web extras components under:", WEB_EXTRAS_LOCAL_PATH
266
267        if "REQUEST_XML_DIR_LOCAL_PATH" in configVarDict.keys():
268            if overwrite!="ON":
269                checkBeforeMakingDir(REQUEST_XML_DIR_LOCAL_PATH, 0777)             
270           
271                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % REQUEST_XML_DIR_LOCAL_PATH)
272                if destroy!="yes":
273                    print "Exiting setup without completing."
274                    sys.exit()   
275            try:
276                os.system("rm -fR %s" % REQUEST_XML_DIR_LOCAL_PATH)
277                os.makedirs(REQUEST_XML_DIR_LOCAL_PATH)
278                print "Made Request XML directory: %s" % REQUEST_XML_DIR_LOCAL_PATH
279                os.chmod(REQUEST_XML_DIR_LOCAL_PATH, 0777)
280                #os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
281            except:
282                raise "Cannot create directory: %s" % REQUEST_XML_DIR_LOCAL_PATH """
283       
284        configMap={"s":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "serverConfig.py"), 
285                   "c":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "clientConfig.py")}
286        print "Renaming configuration variables in config file:"
287       
288        configFileName=configMap[package[-1]]
289        configFile=open(configFileName)
290        configLines=configFile.readlines()
291        configFile.close()
292       
293        shutil.copy(configFileName, configFileName+".bak")
294       
295        newLines=[]
296        for line in configLines:
297            matched=None
298            for confVar, value in configVarDict.items():
299               
300                if re.match(r"\s*%s\s*=" % confVar, line):
301                    newLine="%s=%s\n" % (confVar, value)
302                    print "CHANGING:   %sTO:         %s" % (line, newLine)
303                    newLines.append(newLine)
304                    matched=1
305                    break
306                   
307            if matched==None:
308                newLines.append(line)     
309
310        print len(configLines), len(newLines)
311        configFile=open(configFileName, "w")
312        configFile.write("".join(newLines))
313        configFile.close()
314        print "Re-written config file: %s\n" % (configFileName)
315
316        if package=="dxs":
317            print "Re-naming dataset paths in inputDatasets.xml file...\n"
318            inputPath="DATASET_PATH_TO_CHANGE"
319            outputPath=os.path.join(BASEDIR, "testdata")
320            inputDatasetsFile=os.path.join(BASEDIR, "datasets", "inputDatasets.xml")
321            os.system("perl -p -i -w -e 's!%s!%s!g;' %s*" % (inputPath, outputPath, inputDatasetsFile))
322            print "Done.\n"
323
324        print "Installing python package: %s" % packageName
325
326        os.chdir(os.path.join("..", "%s-%s" % (pypackage, version)))
327        setup(name=pypackage,
328            version=version,
329            description=packageName,
330            author="Ag Stephens",           
331            author_email="a.stephens@rl.ac.uk",
332            url="http://ndg.nerc.ac.uk/DataExtractor",
333            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
334This software may be distributed under the terms of the
335Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
336            packages=[pypackage])       
337   
338    print "\nIt looks like setup has completed successfully!\n"
339    print """You can start the server Web Service by typing:
340    python %s/bin/DXWSInterface.py
341
342You should then be able to view the client package via a browser at:
343    %s
344   
345Have fun!....and of course send any bugs to <a.stephens@rl.ac.uk>
346
347""" % (BASEDIR, CGI_SCRIPT_URL_PATH)
348   
349   
350def testImportExternalPackages():
351    """
352    Tests that external packages can be imported.
353    """
354    print "Testing import of required cdms (CDAT) package...",
355    try:
356        import cdms         
357        print "OK!"
358    except:
359        print """\n\nCannot import cdms package. Please try the one or more of the following to fix this:
360
3611. Set your PYTHONPATH environment variable to pick up the correct directory containing the cdms package.
3622. Retry using the correct python installation such as "<my_python>/bin/python setup.py"
3633. Install CDAT (cdms)."""
364        sys.exit()
365
366    print "Testing import required ZSI package...",
367    try:
368        import ZSI
369        print "OK!"
370    except:
371        print """\n\nCannot import ZSI package. Please try one or more of the following to fix this:
372
3731. Set your PYTHONPATH environment variable to pick up the correct directory containing the ZSI package.
3742. Install ZSI."""
375        sys.exit()
376 
377 
378def checkBeforeMakingDir(dirname, permissions=0755, owner=None, group=None):
379    """
380    Checks with the user before deleting and re-creating directories.
381    """
382    if os.path.isdir(dirname):
383        print "\nThis package needs uses the following directory which already exists:\n",dirname
384        cleanOut=raw_input("Type 'yes' to clean out ALL existing contents of this directory or press return to only overwrite files being added:")
385        if cleanOut=="yes":         
386            removeDirWithCheck(dirname)
387           
388    sep=os.path.sep
389    pathParts=dirname.split(sep)
390    cpath=""
391    for part in pathParts:
392        cpath=cpath+sep+part
393        makeDirAndPerm(dirname, permissions=permissions, owner=owner, group=group)
394    print "Directory is ready: ", dirname       
395
396def removeDirWithCheck(dirname):
397    """
398    Removes dir but asks users permission.
399    """
400    if os.path.isdir(dirname):
401        deleteIt=raw_input("Type 'yes' to confirm you REALLY want to delete the following directory '%s':" % dirname)
402        if deleteIt=="yes":
403            os.system("rm -fR %s" % dirname)
404
405
406def makeDirAndPerm(dirname, permissions=0755, owner=None, group=None):
407    """
408    Tries to make a directory with appropriate permissions, owner etc.
409    """
410    if not os.path.isdir(dirname):
411        os.mkdir(dirname)
412        os.chmod(dirname, permissions)
413        os.system("chown %s %s" % (owner, dirname))
414        os.system("chgrp %s %s" % (group, dirname))
415   
416
417if __name__=="__main__":
418
419    packages=["dxs", "dxc"]       
420
421    if "--overwrite" in sys.argv[1:]:
422        print "Setting overwrite to ON"
423        overwrite="ON"
424        sys.argv.remove("--overwrite")
425       
426    if "--dxs-only" in sys.argv[1:]:
427        sys.argv.remove("--dxs-only")
428        packages=["dxs"]
429    elif "--dxc-only" in sys.argv[1:]:
430        sys.argv.remove("--dxc-only")
431        packages=["dxc"]       
432
433    if "sdist" in sys.argv[1:]:
434        createSourceBundle(packages)
435    else:
436        if "install" not in sys.argv[1:]:
437            print "\nWARNING: Building without installing may mean your version won't pick up the location of the python packages."
438            time.sleep(0.3)   
439        print "\nChecking 'install.conf' configuration file to check you have reset all necessary parameters before beginning."
440        if open("../install.conf").read().find("PLEASE_REPLACE_ME")>-1:
441            print """Please manually edit the 'intall.conf' file and change all parameters that are currently defined as 'PLEASE_REPLACE_ME'.
442           
443This setup script creates (and can overwrite) some directories on your system so it is **CRUCIAL** that you ensure all
444the paths are set appropriately for your system!
445"""         
446        setupPackages(packages)
447       
Note: See TracBrowser for help on using the repository browser.