source: TI03-DataExtractor/trunk/setup.py @ 1715

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/setup.py@1715
Revision 1715, 14.8 KB checked in by astephen, 13 years ago (diff)

Merged with titania version.

Line 
1#!/usr/bin/env python
2
3"""
4setup.py
5========
6
7Set up script for the Data Extractor package.
8
9The Data Extractor is a server package (dxs) that is exposed
10as a Web Service using python's ZSI SOAP Library.
11
12There is also a client package to primarily support the CGI client
13package (dxc).
14
15This installation includes both server and client and assumes you want
16to install both on your local machine. If you only wish to install either
17server or client use one of the following options:
18
19--dxc-only # only install the client package
20--dxs-only # only install the server package
21
22
23Usage:
24======
25
26    [<your_python>/bin/python] setup.py [options] 
27
28[options] can include the above as well as the usual
29
30build | install
31
32Note that using "--overwrite" will automatically overwrite
33existing directories regardless of whether
34
35"""
36
37from distutils.core import setup
38import sys, os, shutil, time, re
39from ConfigParser import ConfigParser
40version="0.3.4"
41overwrite="OFF"
42
43
44def createSourceBundle(packages=["dxs", "dxc"]):
45    """
46    Creates the packages required and includes the directories needed.
47    """
48    basePackage=packages[0][:2]
49    packMap={"s":"Server", "c":"Client"}
50       
51    for package in packages: 
52        if os.path.isfile("MANIFEST"): os.remove("MANIFEST")
53        pypackage="py"+package
54        packageName="Python Data Extractor %s" % packMap[pypackage[-1]]
55        print "\nBundling up source of python package: %s" % packageName
56   
57        setup(name=pypackage,
58            version=version,
59            description=packageName,
60            author="Ag Stephens",           
61            author_email="a.stephens@rl.ac.uk",
62            url="http://ndg.nerc.ac.uk/DataExtractor",
63            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
64This software may be distributed under the terms of the
65Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
66            packages=[pypackage])   
67           
68        print "Bundling up the source of the non-python stuff and python scripts for: %s" % packageName
69        tarfile="%s-%s.tar" % (package, version)
70        gzipfile=tarfile+".gz"
71        os.system("tar -cvf %s %s" % (tarfile, package))
72        os.system("gzip %s" % tarfile)
73       
74        if not os.path.isfile(gzipfile):
75            raise "Did not successfully package up non-python source for '%s'." % package
76           
77        print "Moving gzip file into distribution..."
78        os.rename(gzipfile, os.path.join("dist", gzipfile))
79
80        if package=="dxc":
81            print "Copying cgi directory..."
82            if os.path.isdir("dist/cgi"):
83                if os.path.isfile("dist/cgi/dxui"):
84                    os.remove("dist/cgi/dxui")
85                os.rmdir("dist/cgi")
86            shutil.copytree("cgi", "dist/cgi")
87           
88            print "Copying web_extras directory..."
89            if os.path.isdir("dist/web_extras"):
90                os.system("rm -fR dist/web_extras")
91            shutil.copytree("web_extras", "dist/web_extras")       
92           
93    print "Copying README.txt and install.conf files..."
94    shutil.copy("README.txt", "dist")
95    shutil.copy("install.sh", "dist")   
96    shutil.copy("install.conf", "dist") 
97    shutil.copy("install.conf", "dist/install.conf.orig")
98       
99    print "Tarring up everything in dist directory..."
100    tarAllFile="%s-%s-all.tar" % (basePackage, version)
101    os.system("cd dist ; tar -cvf %s *" % tarAllFile)
102       
103    if not os.path.isfile(os.path.join("dist", tarAllFile)):
104        raise "Did not successfully package up tar-all file '%s'." % tarAllFile
105       
106
107
108def setupPackages(packages=["dxs", "dxc"]):
109    """
110    Controls the entire setup and creates directories and
111    permissions where necessary to get the ancillary stuff working.
112    """   
113    testImportExternalPackages(packages)
114   
115    packMap={"s":"Server", "c":"Client"}
116       
117    for package in packages: 
118        pypackage="py"+package
119        packageName="Python Data Extractor %s" % packMap[package[-1]]
120           
121        print "Installing the non-python stuff and python scripts for: %s" % packageName
122        unpackageDir="%s-%s" % (package, version)
123        tarfile="%s.tar" % unpackageDir
124        gzipfile=tarfile+".gz"
125        os.system("cd .. ; gunzip %s" % tarfile)       
126        os.system("cd .. ; tar -xvf %s" % tarfile)
127
128        packageDir=os.path.join("..", package)
129        if not os.path.isdir(packageDir):
130            raise "Did not successfully unpack the source files for %s." % package
131           
132        print "Reading configuration file to get %s local details." % package
133        config=ConfigParser()
134        config.read("../install.conf")
135 
136        print "Set the configuration variables that are used to generate others..."
137        for item in ["BASEDIR", "STATIC_WEBSERVER_LOCAL_PATH", "STATIC_WEBSERVER_URL_PATH",
138                     "WEB_EXTRAS_URL_PATH"]:
139            if item.lower() in config.options(package):
140                exec("%s=%s" % (item.upper(), config.get(package, item.lower())))
141                print "SET:", item
142 
143        print "\n\n"
144        configVarDict={}
145        for configVar in config.options(package):
146                value=config.get(package, configVar)
147                print "Setting: %s=%s" % (configVar.upper(), value)
148                exec("%s=%s" % (configVar.upper(), value))
149                configVarDict[configVar.upper()]=value
150
151        print "Now we have parsed the config file we can start copying and making the source directories."
152        print "Installing base package '%s' in: %s" % (package, BASEDIR)
153
154        if package=="dxs":
155                dxsBasedir=BASEDIR # for message at end
156
157        if overwrite!="ON":
158            checkBeforeMakingDir(BASEDIR)
159         
160       
161        for item in os.listdir(packageDir):
162            sourceCopy=os.path.join(packageDir, item)
163            target=os.path.join(BASEDIR, item)
164            if os.path.exists(target): 
165                os.system("rm -fr %s" % target)
166           
167           
168            if os.path.isfile(sourceCopy):
169                shutil.copy(sourceCopy, target)
170            elif os.path.isdir(sourceCopy):
171                try:
172                    shutil.copytree(sourceCopy, target)
173                except:
174                    makeDirAndPerm(os.path.split(target)[0])
175                    shutil.copytree(sourceCopy, target)
176       
177        print "Installed non-python package and scripts under:", BASEDIR
178
179        if "OUTPUT_DIR" in configVarDict.keys():
180            if overwrite!="ON":
181                checkBeforeMakingDir(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP)           
182               
183            else:
184                makeDirAndPerm(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP) 
185
186        for dirToMake in ["OUTPUT_DIR_LOCAL_PATH"]:
187         
188          if dirToMake in configVarDict.keys():
189            dirToMake=eval(configVarDict[dirToMake])     
190            if overwrite!="ON":
191                checkBeforeMakingDir(OUTPUT_DIR_LOCAL_PATH)     
192            else:
193                makeDirAndPerm(OUTPUT_DIR_LOCAL_PATH)       
194
195        # Define config files here so that dxui cgi script can have
196        # the config file location inserted...
197        configMap={"s":os.path.join(BASEDIR, "configs", "serverConfig.py"), 
198                   "c":os.path.join(BASEDIR, "configs", "clientConfig.py")}
199                   
200        configFileName=configMap[package[-1]]   
201
202        if "CGI_SCRIPT_LOCAL_PATH" in configVarDict.keys():
203            try:
204                print "\nAdding your python and location of config file to cgi script:", CGI_SCRIPT_LOCAL_PATH
205                shutil.copy("../cgi/dxui", CGI_SCRIPT_LOCAL_PATH)
206                os.chmod(CGI_SCRIPT_LOCAL_PATH, 0755)
207                thisPython=sys.modules["os"].__file__.split("lib")[0]+"bin/python"
208                print thisPython
209                os.system("perl -p -i -w -e 's:<YOUR_PYTHON_HERE>:%s:g;' %s*" % (thisPython, CGI_SCRIPT_LOCAL_PATH))
210                os.system("perl -p -i -w -e 's:<PLACE_CONFIG_PATH_HERE>:%s:g;' %s*" % (configFileName, CGI_SCRIPT_LOCAL_PATH)) 
211
212                print "\nCopied CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
213            except:
214                raise "Cannot move CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
215
216        if "WEB_EXTRAS_LOCAL_PATH" in configVarDict.keys():
217            if overwrite!="ON":
218                checkBeforeMakingDir(WEB_EXTRAS_LOCAL_PATH, 0755)
219            else:
220                makeDirAndPerm(WEB_EXTRAS_LOCAL_PATH, 0755)     
221
222            webExtrasDir="../web_extras"
223            for item in os.listdir(webExtrasDir):
224                sourceCopy=os.path.join(webExtrasDir, item)
225                target=os.path.join(WEB_EXTRAS_LOCAL_PATH, item)
226                if os.path.exists(target): 
227                    os.system("rm -fR %s" % target)
228           
229                if os.path.isfile(sourceCopy):
230                    shutil.copy(sourceCopy, target)
231                elif os.path.isdir(sourceCopy):
232                    shutil.copytree(sourceCopy, target)
233       
234            print "Installed web extras components under:", WEB_EXTRAS_LOCAL_PATH
235
236        if "REQUEST_XML_DIR_LOCAL_PATH" in configVarDict.keys():
237            if overwrite!="ON":
238                checkBeforeMakingDir(REQUEST_XML_DIR_LOCAL_PATH, 0777) 
239            else:
240                makeDirAndPerm(REQUEST_XML_DIR_LOCAL_PATH, 0777)
241           
242        if "SESSION_OBJECT_DIR" in configVarDict.keys():
243            if overwrite!="ON":
244                checkBeforeMakingDir(SESSION_OBJECT_DIR, 0777)
245            else:
246                makeDirAndPerm(SESSION_OBJECT_DIR, 0777)                               
247                   
248        print "Renaming configuration variables in config file:", configFileName
249       
250        configFile=open(configFileName)
251        configLines=configFile.readlines()
252        configFile.close()
253       
254        shutil.copy(configFileName, configFileName+".bak")
255       
256        newLines=[]
257        confFileVarDict={}
258        for line in configLines:
259            matched=None
260            for confVar, value in configVarDict.items():
261               
262                if re.match(r"\s*%s\s*=" % confVar, line):
263                    newLine="%s=%s\n" % (confVar, value)
264                    confFileVarDict[confVar]=value
265                    print "CHANGING:   %sTO:         %s" % (line, newLine)
266                    newLines.append(newLine)
267                    matched=1
268                    break
269                   
270            if matched==None:
271                newLines.append(line)     
272
273        print len(configLines), len(newLines)
274        configFile=open(configFileName, "w")
275        configFile.write("".join(newLines))
276        configFile.close()
277        print "Re-written config file: %s\n" % (configFileName)
278           
279        if "MAP_APPLET_TEMPLATE_LOCAL_DIR" in confFileVarDict.keys():           
280            print "\nChanging permissions on maps/ directory..."
281            os.chmod(MAP_APPLET_TEMPLATE_LOCAL_DIR, 0777)
282
283        # Need my cleaning old python installations code here...
284
285        if package=="dxs":
286            print "Re-naming dataset paths in inputDatasets.xml file...\n"
287            inputPath="DATASET_PATH_TO_CHANGE"
288            outputPath=os.path.join(BASEDIR, "testdata")
289            inputDatasetsFile=os.path.join(BASEDIR, "datasets", "inputDatasets.xml")
290            os.system("perl -p -i -w -e 's!%s!%s!g;' %s*" % (inputPath, outputPath, inputDatasetsFile))
291            print "Done.\n"
292
293        print "Installing python package: %s" % packageName
294
295        os.chdir(os.path.join("..", "%s-%s" % (pypackage, version)))
296        setup(name=pypackage,
297            version=version,
298            description=packageName,
299            author="Ag Stephens",           
300            author_email="a.stephens@rl.ac.uk",
301            url="http://ndg.nerc.ac.uk/DataExtractor",
302            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
303This software may be distributed under the terms of the
304Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
305            packages=[pypackage])       
306   
307    print "\nIt looks like setup has completed successfully!\n"
308    if "dxs" in packages:
309        print """You can start the server Web Service by typing:
310    python %s/bin/DXWSInterface.py %s/configs/serverConfig.py
311    """ % (dxsBasedir, dxsBasedir)
312
313    if "dxc" in packages:
314        print """You should then be able to view the client package via a browser at:
315    %s
316    """ % CGI_SCRIPT_URL_PATH
317   
318    print """
319Have fun!....and of course send any bugs to <a.stephens@rl.ac.uk>
320
321""" 
322   
323   
324def testImportExternalPackages(packages):
325    """
326    Tests that external packages can be imported.
327    """
328    if "dxs" in packages:
329        print "Testing import of required cdms (CDAT) package...",
330        try:
331            import cdms         
332            print "OK!"
333        except:
334            print """\n\nCannot import cdms package. Please try the one or more of the following to fix this:
335
3361. Set your PYTHONPATH environment variable to pick up the correct directory containing the cdms package.
3372. Retry using the correct python installation such as "<my_python>/bin/python setup.py"
3383. Install CDAT (cdms)."""
339            sys.exit()
340
341    print "Testing import required ZSI package...",
342    try:
343        import ZSI
344
345        print "OK!"
346    except:
347        print """\n\nCannot import ZSI package. Please try one or more of the following to fix this:
348
3491. Set your PYTHONPATH environment variable to pick up the correct directory containing the ZSI package.
3502. Install ZSI."""
351        sys.exit()
352 
353 
354def checkBeforeMakingDir(dirname, permissions=0755, owner=None, group=None):
355    """
356   Checks with the user before deleting and re-creating directories.
357   """
358    if os.path.isdir(dirname):
359        print "\nThis package needs uses the following directory which already exists:\n",dirname
360        cleanOut=raw_input("Type 'yes' to clean out ALL existing contents of this directory or press return to only overwrite files being added:")
361        if cleanOut=="yes":         
362            removeDirWithCheck(dirname)
363           
364    sep=os.path.sep
365    pathParts=dirname.split(sep)
366    cpath=""
367    for part in pathParts:
368        cpath=cpath+sep+part
369        makeDirAndPerm(dirname, permissions=permissions, owner=owner, group=group)
370    print "Directory is ready: ", dirname       
371
372
373
374def removeDirWithCheck(dirname):
375    """
376    Removes dir but asks users permission.
377    """
378    if os.path.isdir(dirname):
379        deleteIt=raw_input("Type 'yes' to confirm you REALLY want to delete the following directory '%s':" % dirname)
380        if deleteIt=="yes":
381            os.system("rm -fR %s" % dirname)
382
383
384def makeDirAndPerm(dirname, permissions=0755, owner=None, group=None):
385    """
386    Tries to make a directory with appropriate permissions, owner etc.
387    """
388    if not os.path.isdir(dirname):
389        parentDir=os.path.split(dirname)[0]
390        if not os.path.isdir(parentDir):
391            makeDirAndPerm(parentDir)
392        os.mkdir(dirname)
393        os.chmod(dirname, permissions)
394        if owner!=None:
395            os.system("chown %s %s" % (owner, dirname))
396        if group!=None:
397            os.system("chgrp %s %s" % (group, dirname))
398   
399
400if __name__=="__main__":
401
402    packages=["dxs", "dxc"]       
403
404    if "--overwrite" in sys.argv[1:]:
405        print "Setting overwrite to ON"
406        overwrite="ON"
407        sys.argv.remove("--overwrite")
408       
409    if "--dxs-only" in sys.argv[1:]:
410        sys.argv.remove("--dxs-only")
411        packages=["dxs"]
412    elif "--dxc-only" in sys.argv[1:]:
413        sys.argv.remove("--dxc-only")
414        packages=["dxc"]       
415
416    if "sdist" in sys.argv[1:]:
417        createSourceBundle(packages)
418    else:
419        if "install" not in sys.argv[1:]:
420            print "\nWARNING: Building without installing may mean your version won't pick up the location of the python packages."
421            time.sleep(0.3)   
422        print "\nChecking 'install.conf' configuration file to check you have reset all necessary parameters before beginning."
423        if open("../install.conf").read().find("PLEASE_REPLACE_ME")>-1:
424            print """Please manually edit the 'intall.conf' file and change all parameters that are currently defined as 'PLEASE_REPLACE_ME'.
425           
426This setup script creates (and can overwrite) some directories on your system so it is **CRUCIAL** that you ensure all
427the paths are set appropriately for your system!
428"""         
429        setupPackages(packages)
430       
Note: See TracBrowser for help on using the repository browser.