source: TI03-DataExtractor/trunk/setup.py @ 1225

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/setup.py@1225
Revision 1225, 14.2 KB checked in by astephen, 14 years ago (diff)

Latest version from laptop.
Not yet merged with glue branch.

Line 
1#!/usr/bin/env python
2
3"""
4setup.py
5========
6
7Set up script for the Data Extractor package.
8
9The Data Extractor is a server package (dxs) that is exposed
10as a Web Service using python's ZSI SOAP Library.
11
12There is also a client package to primarily support the CGI client
13package (dxc).
14
15This installation includes both server and client and assumes you want
16to install both on your local machine. If you only wish to install either
17server or client use one of the following options:
18
19--dxc-only # only install the client package
20--dxs-only # only install the server package
21
22
23Usage:
24======
25
26    [<your_python>/bin/python] setup.py [options] 
27
28[options] can include the above as well as the usual
29
30build | install
31
32Note that using "--overwrite" will automatically overwrite
33existing directories regardless of whether
34
35"""
36
37from distutils.core import setup
38import sys, os, shutil, time, re
39from ConfigParser import ConfigParser
40version="0.3.0"
41overwrite="OFF"
42
43
44def createSourceBundle(packages=["dxs", "dxc"]):
45    """
46    Creates the packages required and includes the directories needed.
47    """
48    basePackage=packages[0][:2]
49    packMap={"s":"Server", "c":"Client"}
50       
51    for package in packages: 
52        if os.path.isfile("MANIFEST"): os.remove("MANIFEST")
53        pypackage="py"+package
54        packageName="Python Data Extractor %s" % packMap[pypackage[-1]]
55        print "\nBundling up source of python package: %s" % packageName
56   
57        setup(name=pypackage,
58            version=version,
59            description=packageName,
60            author="Ag Stephens",           
61            author_email="a.stephens@rl.ac.uk",
62            url="http://ndg.nerc.ac.uk/DataExtractor",
63            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
64This software may be distributed under the terms of the
65Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
66            packages=[pypackage])   
67           
68        print "Bundling up the source of the non-python stuff and python scripts for: %s" % packageName
69        tarfile="%s-%s.tar" % (package, version)
70        gzipfile=tarfile+".gz"
71        os.system("tar -cvf %s %s" % (tarfile, package))
72        os.system("gzip %s" % tarfile)
73       
74        if not os.path.isfile(gzipfile):
75            raise "Did not successfully package up non-python source for '%s'." % package
76           
77        print "Moving gzip file into distribution..."
78        os.rename(gzipfile, os.path.join("dist", gzipfile))
79
80        if package=="dxc":
81            print "Copying cgi directory..."
82            if os.path.isdir("dist/cgi"):
83                if os.path.isfile("dist/cgi/dxui"):
84                    os.remove("dist/cgi/dxui")
85                os.rmdir("dist/cgi")
86            shutil.copytree("cgi", "dist/cgi")
87           
88            print "Copying web_extras directory..."
89            if os.path.isdir("dist/web_extras"):
90                os.system("rm -fR dist/web_extras")
91            shutil.copytree("web_extras", "dist/web_extras")       
92           
93    print "Copying README.txt and install.conf files..."
94    shutil.copy("README.txt", "dist")
95    shutil.copy("install.sh", "dist")   
96    shutil.copy("install.conf", "dist") 
97    shutil.copy("install.conf", "dist/install.conf.orig")
98       
99    print "Tarring up everything in dist directory..."
100    tarAllFile="%s-%s-all.tar" % (basePackage, version)
101    os.system("cd dist ; tar -cvf %s *" % tarAllFile)
102       
103    if not os.path.isfile(os.path.join("dist", tarAllFile)):
104        raise "Did not successfully package up tar-all file '%s'." % tarAllFile
105       
106
107
108def setupPackages(packages=["dxs", "dxc"]):
109    """
110    Controls the entire setup and creates directories and
111    permissions where necessary to get the ancillary stuff working.
112    """   
113    testImportExternalPackages(packages)
114   
115    packMap={"s":"Server", "c":"Client"}
116       
117    for package in packages: 
118        pypackage="py"+package
119        packageName="Python Data Extractor %s" % packMap[package[-1]]
120           
121        print "Installing the non-python stuff and python scripts for: %s" % packageName
122        unpackageDir="%s-%s" % (package, version)
123        tarfile="%s.tar" % unpackageDir
124        gzipfile=tarfile+".gz"
125        os.system("cd .. ; gunzip %s" % tarfile)       
126        os.system("cd .. ; tar -xvf %s" % tarfile)
127
128        packageDir=os.path.join("..", package)
129        if not os.path.isdir(packageDir):
130            raise "Did not successfully unpack the source files for %s." % package
131           
132        print "Reading configuration file to get %s local details." % package
133        config=ConfigParser()
134        config.read("../install.conf")
135 
136        print "Set the configuration variables that are used to generate others..."
137        for item in ["BASEDIR", "STATIC_WEBSERVER_LOCAL_PATH", "STATIC_WEBSERVER_URL_PATH",
138                     "WEB_EXTRAS_URL_PATH"]:
139            if item.lower() in config.options(package):
140                exec("%s=%s" % (item.upper(), config.get(package, item.lower())))
141                print "SET:", item
142 
143        print "\n\n"
144        configVarDict={}
145        for configVar in config.options(package):
146                value=config.get(package, configVar)
147                print "Setting: %s=%s" % (configVar.upper(), value)
148                exec("%s=%s" % (configVar.upper(), value))
149                configVarDict[configVar.upper()]=value
150
151        print "Now we have parsed the config file we can start copying and making the source directories."
152        print "Installing base package '%s' in: %s" % (package, BASEDIR)
153
154        if package=="dxs":
155                dxsBasedir=BASEDIR # for message at end
156
157        if overwrite!="ON":
158            checkBeforeMakingDir(BASEDIR)
159         
160       
161        for item in os.listdir(packageDir):
162            sourceCopy=os.path.join(packageDir, item)
163            target=os.path.join(BASEDIR, item)
164            if os.path.exists(target): 
165                os.system("rm -fr %s" % target)
166           
167            if os.path.isfile(sourceCopy):
168                shutil.copy(sourceCopy, target)
169            elif os.path.isdir(sourceCopy):
170                shutil.copytree(sourceCopy, target)
171       
172        print "Installed non-python package and scripts under:", BASEDIR
173
174        if "OUTPUT_DIR" in configVarDict.keys():
175            if overwrite!="ON":
176                checkBeforeMakingDir(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP)           
177               
178            else:
179                makeDirAndPerm(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP) 
180
181        for dirToMake in ["OUTPUT_DIR_LOCAL_PATH"]:
182         
183          if dirToMake in configVarDict.keys():
184            dirToMake=eval(configVarDict[dirToMake])     
185            if overwrite!="ON":
186                checkBeforeMakingDir(OUTPUT_DIR_LOCAL_PATH)     
187            else:
188                makeDirAndPerm(OUTPUT_DIR_LOCAL_PATH)       
189
190        if "CGI_SCRIPT_LOCAL_PATH" in configVarDict.keys():
191            try:
192                shutil.copy("../cgi/dxui", CGI_SCRIPT_LOCAL_PATH)
193                os.chmod(CGI_SCRIPT_LOCAL_PATH, 0755)
194                thisPython=sys.modules["os"].__file__.split("lib")[0]+"bin/python"
195                print thisPython
196                os.system("perl -p -i -w -e 's:<YOUR_PYTHON_HERE>:%s:g;' %s*" % (thisPython, CGI_SCRIPT_LOCAL_PATH))
197
198                print "\nCopied CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
199            except:
200                raise "Cannot move CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
201
202        if "WEB_EXTRAS_LOCAL_PATH" in configVarDict.keys():
203            if overwrite!="ON":
204                checkBeforeMakingDir(WEB_EXTRAS_LOCAL_PATH, 0755)
205            else:
206                makeDirAndPerm(WEB_EXTRAS_LOCAL_PATH, 0755)     
207
208            webExtrasDir="../web_extras"
209            for item in os.listdir(webExtrasDir):
210                sourceCopy=os.path.join(webExtrasDir, item)
211                target=os.path.join(WEB_EXTRAS_LOCAL_PATH, item)
212                if os.path.exists(target): 
213                    os.system("rm -fR %s" % target)
214           
215                if os.path.isfile(sourceCopy):
216                    shutil.copy(sourceCopy, target)
217                elif os.path.isdir(sourceCopy):
218                    shutil.copytree(sourceCopy, target)
219       
220            print "Installed web extras components under:", WEB_EXTRAS_LOCAL_PATH
221
222        if "REQUEST_XML_DIR_LOCAL_PATH" in configVarDict.keys():
223            if overwrite!="ON":
224                checkBeforeMakingDir(REQUEST_XML_DIR_LOCAL_PATH, 0777) 
225            else:
226                makeDirAndPerm(REQUEST_XML_DIR_LOCAL_PATH, 0777)
227           
228        if "SESSION_OBJECT_DIR" in configVarDict.keys():
229            if overwrite!="ON":
230                checkBeforeMakingDir(SESSION_OBJECT_DIR, 0777)
231            else:
232                makeDirAndPerm(SESSION_OBJECT_DIR, 0777)                               
233       
234        if "MAP_APPLET_TEMPLATE_LOCAL_PATH" in configVarDict.keys():
235            print "\nChanging permissions on maps/ directory..."
236            os.chmod(os.path.split(MAP_APPLET_TEMPLATE_LOCAL_PATH)[0], 0777)
237       
238        configMap={"s":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "serverConfig.py"), 
239                   "c":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "clientConfig.py")}
240        print "Renaming configuration variables in config file:"
241       
242        configFileName=configMap[package[-1]]
243        configFile=open(configFileName)
244        configLines=configFile.readlines()
245        configFile.close()
246       
247        shutil.copy(configFileName, configFileName+".bak")
248       
249        newLines=[]
250        for line in configLines:
251            matched=None
252            for confVar, value in configVarDict.items():
253               
254                if re.match(r"\s*%s\s*=" % confVar, line):
255                    newLine="%s=%s\n" % (confVar, value)
256                    print "CHANGING:   %sTO:         %s" % (line, newLine)
257                    newLines.append(newLine)
258                    matched=1
259                    break
260                   
261            if matched==None:
262                newLines.append(line)     
263
264        print len(configLines), len(newLines)
265        configFile=open(configFileName, "w")
266        configFile.write("".join(newLines))
267        configFile.close()
268        print "Re-written config file: %s\n" % (configFileName)
269
270        if package=="dxs":
271            print "Re-naming dataset paths in inputDatasets.xml file...\n"
272            inputPath="DATASET_PATH_TO_CHANGE"
273            outputPath=os.path.join(BASEDIR, "testdata")
274            inputDatasetsFile=os.path.join(BASEDIR, "datasets", "inputDatasets.xml")
275            os.system("perl -p -i -w -e 's!%s!%s!g;' %s*" % (inputPath, outputPath, inputDatasetsFile))
276            print "Done.\n"
277
278        print "Installing python package: %s" % packageName
279
280        os.chdir(os.path.join("..", "%s-%s" % (pypackage, version)))
281        setup(name=pypackage,
282            version=version,
283            description=packageName,
284            author="Ag Stephens",           
285            author_email="a.stephens@rl.ac.uk",
286            url="http://ndg.nerc.ac.uk/DataExtractor",
287            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
288This software may be distributed under the terms of the
289Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
290            packages=[pypackage])       
291   
292    print "\nIt looks like setup has completed successfully!\n"
293    if "dxs" in packages:
294        print """You can start the server Web Service by typing:
295    python %s/bin/DXWSInterface.py
296    """ % dxsBasedir
297
298    if "dxc" in packages:
299        print """You should then be able to view the client package via a browser at:
300    %s
301    """ % CGI_SCRIPT_URL_PATH
302   
303    print """
304Have fun!....and of course send any bugs to <a.stephens@rl.ac.uk>
305
306""" 
307   
308   
309def testImportExternalPackages(packages):
310    """
311    Tests that external packages can be imported.
312    """
313    if "dxs" in packages:
314        print "Testing import of required cdms (CDAT) package...",
315        try:
316            import cdms         
317            print "OK!"
318        except:
319            print """\n\nCannot import cdms package. Please try the one or more of the following to fix this:
320
3211. Set your PYTHONPATH environment variable to pick up the correct directory containing the cdms package.
3222. Retry using the correct python installation such as "<my_python>/bin/python setup.py"
3233. Install CDAT (cdms)."""
324            sys.exit()
325
326    print "Testing import required ZSI package...",
327    try:
328        import ZSI
329        print "OK!"
330    except:
331        print """\n\nCannot import ZSI package. Please try one or more of the following to fix this:
332
3331. Set your PYTHONPATH environment variable to pick up the correct directory containing the ZSI package.
3342. Install ZSI."""
335        sys.exit()
336 
337 
338def checkBeforeMakingDir(dirname, permissions=0755, owner=None, group=None):
339    """
340   Checks with the user before deleting and re-creating directories.
341   """
342    if os.path.isdir(dirname):
343        print "\nThis package needs uses the following directory which already exists:\n",dirname
344        cleanOut=raw_input("Type 'yes' to clean out ALL existing contents of this directory or press return to only overwrite files being added:")
345        if cleanOut=="yes":         
346            removeDirWithCheck(dirname)
347           
348    sep=os.path.sep
349    pathParts=dirname.split(sep)
350    cpath=""
351    for part in pathParts:
352        cpath=cpath+sep+part
353        makeDirAndPerm(dirname, permissions=permissions, owner=owner, group=group)
354    print "Directory is ready: ", dirname       
355
356
357def removeDirWithCheck(dirname):
358    """
359    Removes dir but asks users permission.
360    """
361    if os.path.isdir(dirname):
362        deleteIt=raw_input("Type 'yes' to confirm you REALLY want to delete the following directory '%s':" % dirname)
363        if deleteIt=="yes":
364            os.system("rm -fR %s" % dirname)
365
366
367def makeDirAndPerm(dirname, permissions=0755, owner=None, group=None):
368    """
369    Tries to make a directory with appropriate permissions, owner etc.
370    """
371    if not os.path.isdir(dirname):
372        parentDir=os.path.split(dirname)[0]
373        if not os.path.isdir(parentDir):
374            makeDirAndPerm(parentDir)
375        os.mkdir(dirname)
376        os.chmod(dirname, permissions)
377        if owner!=None:
378            os.system("chown %s %s" % (owner, dirname))
379        if group!=None:
380            os.system("chgrp %s %s" % (group, dirname))
381   
382
383if __name__=="__main__":
384
385    packages=["dxs", "dxc"]       
386
387    if "--overwrite" in sys.argv[1:]:
388        print "Setting overwrite to ON"
389        overwrite="ON"
390        sys.argv.remove("--overwrite")
391       
392    if "--dxs-only" in sys.argv[1:]:
393        sys.argv.remove("--dxs-only")
394        packages=["dxs"]
395    elif "--dxc-only" in sys.argv[1:]:
396        sys.argv.remove("--dxc-only")
397        packages=["dxc"]       
398
399    if "sdist" in sys.argv[1:]:
400        createSourceBundle(packages)
401    else:
402        if "install" not in sys.argv[1:]:
403            print "\nWARNING: Building without installing may mean your version won't pick up the location of the python packages."
404            time.sleep(0.3)   
405        print "\nChecking 'install.conf' configuration file to check you have reset all necessary parameters before beginning."
406        if open("../install.conf").read().find("PLEASE_REPLACE_ME")>-1:
407            print """Please manually edit the 'intall.conf' file and change all parameters that are currently defined as 'PLEASE_REPLACE_ME'.
408           
409This setup script creates (and can overwrite) some directories on your system so it is **CRUCIAL** that you ensure all
410the paths are set appropriately for your system!
411"""         
412        setupPackages(packages)
413       
Note: See TracBrowser for help on using the repository browser.