source: TI03-DataExtractor/trunk/setup.py @ 1244

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/setup.py@1244
Revision 1244, 14.7 KB checked in by astephen, 13 years ago (diff)

Close to alpha version.

Line 
1#!/usr/bin/env python
2
3"""
4setup.py
5========
6
7Set up script for the Data Extractor package.
8
9The Data Extractor is a server package (dxs) that is exposed
10as a Web Service using python's ZSI SOAP Library.
11
12There is also a client package to primarily support the CGI client
13package (dxc).
14
15This installation includes both server and client and assumes you want
16to install both on your local machine. If you only wish to install either
17server or client use one of the following options:
18
19--dxc-only # only install the client package
20--dxs-only # only install the server package
21
22
23Usage:
24======
25
26    [<your_python>/bin/python] setup.py [options] 
27
28[options] can include the above as well as the usual
29
30build | install
31
32Note that using "--overwrite" will automatically overwrite
33existing directories regardless of whether
34
35"""
36
37from distutils.core import setup
38import sys, os, shutil, time, re
39from ConfigParser import ConfigParser
40version="0.3.0"
41overwrite="OFF"
42
43
44def createSourceBundle(packages=["dxs", "dxc"]):
45    """
46    Creates the packages required and includes the directories needed.
47    """
48    basePackage=packages[0][:2]
49    packMap={"s":"Server", "c":"Client"}
50       
51    for package in packages: 
52        if os.path.isfile("MANIFEST"): os.remove("MANIFEST")
53        pypackage="py"+package
54        packageName="Python Data Extractor %s" % packMap[pypackage[-1]]
55        print "\nBundling up source of python package: %s" % packageName
56   
57        setup(name=pypackage,
58            version=version,
59            description=packageName,
60            author="Ag Stephens",           
61            author_email="a.stephens@rl.ac.uk",
62            url="http://ndg.nerc.ac.uk/DataExtractor",
63            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
64This software may be distributed under the terms of the
65Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
66            packages=[pypackage])   
67           
68        print "Bundling up the source of the non-python stuff and python scripts for: %s" % packageName
69        tarfile="%s-%s.tar" % (package, version)
70        gzipfile=tarfile+".gz"
71        os.system("tar -cvf %s %s" % (tarfile, package))
72        os.system("gzip %s" % tarfile)
73       
74        if not os.path.isfile(gzipfile):
75            raise "Did not successfully package up non-python source for '%s'." % package
76           
77        print "Moving gzip file into distribution..."
78        os.rename(gzipfile, os.path.join("dist", gzipfile))
79
80        if package=="dxc":
81            print "Copying cgi directory..."
82            if os.path.isdir("dist/cgi"):
83                if os.path.isfile("dist/cgi/dxui"):
84                    os.remove("dist/cgi/dxui")
85                os.rmdir("dist/cgi")
86            shutil.copytree("cgi", "dist/cgi")
87           
88            print "Copying web_extras directory..."
89            if os.path.isdir("dist/web_extras"):
90                os.system("rm -fR dist/web_extras")
91            shutil.copytree("web_extras", "dist/web_extras")       
92           
93    print "Copying README.txt and install.conf files..."
94    shutil.copy("README.txt", "dist")
95    shutil.copy("install.sh", "dist")   
96    shutil.copy("install.conf", "dist") 
97    shutil.copy("install.conf", "dist/install.conf.orig")
98       
99    print "Tarring up everything in dist directory..."
100    tarAllFile="%s-%s-all.tar" % (basePackage, version)
101    os.system("cd dist ; tar -cvf %s *" % tarAllFile)
102       
103    if not os.path.isfile(os.path.join("dist", tarAllFile)):
104        raise "Did not successfully package up tar-all file '%s'." % tarAllFile
105       
106
107
108def setupPackages(packages=["dxs", "dxc"]):
109    """
110    Controls the entire setup and creates directories and
111    permissions where necessary to get the ancillary stuff working.
112    """   
113    testImportExternalPackages(packages)
114   
115    packMap={"s":"Server", "c":"Client"}
116       
117    for package in packages: 
118        pypackage="py"+package
119        packageName="Python Data Extractor %s" % packMap[package[-1]]
120           
121        print "Installing the non-python stuff and python scripts for: %s" % packageName
122        unpackageDir="%s-%s" % (package, version)
123        tarfile="%s.tar" % unpackageDir
124        gzipfile=tarfile+".gz"
125        os.system("cd .. ; gunzip %s" % tarfile)       
126        os.system("cd .. ; tar -xvf %s" % tarfile)
127
128        packageDir=os.path.join("..", package)
129        if not os.path.isdir(packageDir):
130            raise "Did not successfully unpack the source files for %s." % package
131           
132        print "Reading configuration file to get %s local details." % package
133        config=ConfigParser()
134        config.read("../install.conf")
135 
136        print "Set the configuration variables that are used to generate others..."
137        for item in ["BASEDIR", "STATIC_WEBSERVER_LOCAL_PATH", "STATIC_WEBSERVER_URL_PATH",
138                     "WEB_EXTRAS_URL_PATH"]:
139            if item.lower() in config.options(package):
140                exec("%s=%s" % (item.upper(), config.get(package, item.lower())))
141                print "SET:", item
142 
143        print "\n\n"
144        configVarDict={}
145        for configVar in config.options(package):
146                value=config.get(package, configVar)
147                print "Setting: %s=%s" % (configVar.upper(), value)
148                exec("%s=%s" % (configVar.upper(), value))
149                configVarDict[configVar.upper()]=value
150
151        print "Now we have parsed the config file we can start copying and making the source directories."
152        print "Installing base package '%s' in: %s" % (package, BASEDIR)
153
154        if package=="dxs":
155                dxsBasedir=BASEDIR # for message at end
156
157        if overwrite!="ON":
158            checkBeforeMakingDir(BASEDIR)
159         
160       
161        for item in os.listdir(packageDir):
162            sourceCopy=os.path.join(packageDir, item)
163            target=os.path.join(BASEDIR, item)
164            if os.path.exists(target): 
165                os.system("rm -fr %s" % target)
166           
167           
168            if os.path.isfile(sourceCopy):
169                shutil.copy(sourceCopy, target)
170            elif os.path.isdir(sourceCopy):
171                try:
172                    shutil.copytree(sourceCopy, target)
173                except:
174                    makeDirAndPerm(os.path.split(target)[0])
175                    shutil.copytree(sourceCopy, target)
176       
177        print "Installed non-python package and scripts under:", BASEDIR
178
179        if "OUTPUT_DIR" in configVarDict.keys():
180            if overwrite!="ON":
181                checkBeforeMakingDir(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP)           
182               
183            else:
184                makeDirAndPerm(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP) 
185
186        for dirToMake in ["OUTPUT_DIR_LOCAL_PATH"]:
187         
188          if dirToMake in configVarDict.keys():
189            dirToMake=eval(configVarDict[dirToMake])     
190            if overwrite!="ON":
191                checkBeforeMakingDir(OUTPUT_DIR_LOCAL_PATH)     
192            else:
193                makeDirAndPerm(OUTPUT_DIR_LOCAL_PATH)       
194
195        # Define config files here so that dxui cgi script can have
196        # the config file location inserted...
197        configMap={"s":os.path.join(BASEDIR, "configs", "serverConfig.py"), 
198                   "c":os.path.join(BASEDIR, "configs", "clientConfig.py")}
199                   
200        configFileName=configMap[package[-1]]   
201
202        if "CGI_SCRIPT_LOCAL_PATH" in configVarDict.keys():
203            try:
204                print "\nAdding your python and location of config file to cgi script:", CGI_SCRIPT_LOCAL_PATH
205                shutil.copy("../cgi/dxui", CGI_SCRIPT_LOCAL_PATH)
206                os.chmod(CGI_SCRIPT_LOCAL_PATH, 0755)
207                thisPython=sys.modules["os"].__file__.split("lib")[0]+"bin/python"
208                print thisPython
209                os.system("perl -p -i -w -e 's:<YOUR_PYTHON_HERE>:%s:g;' %s*" % (thisPython, CGI_SCRIPT_LOCAL_PATH))
210                os.system("perl -p -i -w -e 's:<PLACE_CONFIG_PATH_HERE>:%s:g;' %s*" % (configFileName, CGI_SCRIPT_LOCAL_PATH)) 
211
212                print "\nCopied CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
213            except:
214                raise "Cannot move CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
215
216        if "WEB_EXTRAS_LOCAL_PATH" in configVarDict.keys():
217            if overwrite!="ON":
218                checkBeforeMakingDir(WEB_EXTRAS_LOCAL_PATH, 0755)
219            else:
220                makeDirAndPerm(WEB_EXTRAS_LOCAL_PATH, 0755)     
221
222            webExtrasDir="../web_extras"
223            for item in os.listdir(webExtrasDir):
224                sourceCopy=os.path.join(webExtrasDir, item)
225                target=os.path.join(WEB_EXTRAS_LOCAL_PATH, item)
226                if os.path.exists(target): 
227                    os.system("rm -fR %s" % target)
228           
229                if os.path.isfile(sourceCopy):
230                    shutil.copy(sourceCopy, target)
231                elif os.path.isdir(sourceCopy):
232                    shutil.copytree(sourceCopy, target)
233       
234            print "Installed web extras components under:", WEB_EXTRAS_LOCAL_PATH
235
236        if "REQUEST_XML_DIR_LOCAL_PATH" in configVarDict.keys():
237            if overwrite!="ON":
238                checkBeforeMakingDir(REQUEST_XML_DIR_LOCAL_PATH, 0777) 
239            else:
240                makeDirAndPerm(REQUEST_XML_DIR_LOCAL_PATH, 0777)
241           
242        if "SESSION_OBJECT_DIR" in configVarDict.keys():
243            if overwrite!="ON":
244                checkBeforeMakingDir(SESSION_OBJECT_DIR, 0777)
245            else:
246                makeDirAndPerm(SESSION_OBJECT_DIR, 0777)                               
247                   
248        print "Renaming configuration variables in config file:", configFileName
249       
250        configFile=open(configFileName)
251        configLines=configFile.readlines()
252        configFile.close()
253       
254        shutil.copy(configFileName, configFileName+".bak")
255       
256        newLines=[]
257        confFileVarDict={}
258        for line in configLines:
259            matched=None
260            for confVar, value in configVarDict.items():
261               
262                if re.match(r"\s*%s\s*=" % confVar, line):
263                    newLine="%s=%s\n" % (confVar, value)
264                    confFileVarDict[confVar]=value
265                    print "CHANGING:   %sTO:         %s" % (line, newLine)
266                    newLines.append(newLine)
267                    matched=1
268                    break
269                   
270            if matched==None:
271                newLines.append(line)     
272
273        print len(configLines), len(newLines)
274        configFile=open(configFileName, "w")
275        configFile.write("".join(newLines))
276        configFile.close()
277        print "Re-written config file: %s\n" % (configFileName)
278
279       
280        if "MAP_APPLET_TEMPLATE_LOCAL_PATH" in confFileVarDict.keys():
281            print "\nChanging permissions on maps/ directory..."
282            os.chmod(os.path.split(MAP_APPLET_TEMPLATE_LOCAL_PATH)[0], 0777)
283
284        # Need my cleaning old python installations code here...
285
286        if package=="dxs":
287            print "Re-naming dataset paths in inputDatasets.xml file...\n"
288            inputPath="DATASET_PATH_TO_CHANGE"
289            outputPath=os.path.join(BASEDIR, "testdata")
290            inputDatasetsFile=os.path.join(BASEDIR, "datasets", "inputDatasets.xml")
291            os.system("perl -p -i -w -e 's!%s!%s!g;' %s*" % (inputPath, outputPath, inputDatasetsFile))
292            print "Done.\n"
293
294        print "Installing python package: %s" % packageName
295
296        os.chdir(os.path.join("..", "%s-%s" % (pypackage, version)))
297        setup(name=pypackage,
298            version=version,
299            description=packageName,
300            author="Ag Stephens",           
301            author_email="a.stephens@rl.ac.uk",
302            url="http://ndg.nerc.ac.uk/DataExtractor",
303            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
304This software may be distributed under the terms of the
305Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
306            packages=[pypackage])       
307   
308    print "\nIt looks like setup has completed successfully!\n"
309    if "dxs" in packages:
310        print """You can start the server Web Service by typing:
311    python %s/bin/DXWSInterface.py
312    """ % dxsBasedir
313
314    if "dxc" in packages:
315        print """You should then be able to view the client package via a browser at:
316    %s
317    """ % CGI_SCRIPT_URL_PATH
318   
319    print """
320Have fun!....and of course send any bugs to <a.stephens@rl.ac.uk>
321
322""" 
323   
324   
325def testImportExternalPackages(packages):
326    """
327    Tests that external packages can be imported.
328    """
329    if "dxs" in packages:
330        print "Testing import of required cdms (CDAT) package...",
331        try:
332            import cdms         
333            print "OK!"
334        except:
335            print """\n\nCannot import cdms package. Please try the one or more of the following to fix this:
336
3371. Set your PYTHONPATH environment variable to pick up the correct directory containing the cdms package.
3382. Retry using the correct python installation such as "<my_python>/bin/python setup.py"
3393. Install CDAT (cdms)."""
340            sys.exit()
341
342    print "Testing import required ZSI package...",
343    try:
344        import ZSI
345        print "OK!"
346    except:
347        print """\n\nCannot import ZSI package. Please try one or more of the following to fix this:
348
3491. Set your PYTHONPATH environment variable to pick up the correct directory containing the ZSI package.
3502. Install ZSI."""
351        sys.exit()
352 
353 
354def checkBeforeMakingDir(dirname, permissions=0755, owner=None, group=None):
355    """
356   Checks with the user before deleting and re-creating directories.
357   """
358    if os.path.isdir(dirname):
359        print "\nThis package needs uses the following directory which already exists:\n",dirname
360        cleanOut=raw_input("Type 'yes' to clean out ALL existing contents of this directory or press return to only overwrite files being added:")
361        if cleanOut=="yes":         
362            removeDirWithCheck(dirname)
363           
364    sep=os.path.sep
365    pathParts=dirname.split(sep)
366    cpath=""
367    for part in pathParts:
368        cpath=cpath+sep+part
369        makeDirAndPerm(dirname, permissions=permissions, owner=owner, group=group)
370    print "Directory is ready: ", dirname       
371
372
373def removeDirWithCheck(dirname):
374    """
375    Removes dir but asks users permission.
376    """
377    if os.path.isdir(dirname):
378        deleteIt=raw_input("Type 'yes' to confirm you REALLY want to delete the following directory '%s':" % dirname)
379        if deleteIt=="yes":
380            os.system("rm -fR %s" % dirname)
381
382
383def makeDirAndPerm(dirname, permissions=0755, owner=None, group=None):
384    """
385    Tries to make a directory with appropriate permissions, owner etc.
386    """
387    if not os.path.isdir(dirname):
388        parentDir=os.path.split(dirname)[0]
389        if not os.path.isdir(parentDir):
390            makeDirAndPerm(parentDir)
391        os.mkdir(dirname)
392        os.chmod(dirname, permissions)
393        if owner!=None:
394            os.system("chown %s %s" % (owner, dirname))
395        if group!=None:
396            os.system("chgrp %s %s" % (group, dirname))
397   
398
399if __name__=="__main__":
400
401    packages=["dxs", "dxc"]       
402
403    if "--overwrite" in sys.argv[1:]:
404        print "Setting overwrite to ON"
405        overwrite="ON"
406        sys.argv.remove("--overwrite")
407       
408    if "--dxs-only" in sys.argv[1:]:
409        sys.argv.remove("--dxs-only")
410        packages=["dxs"]
411    elif "--dxc-only" in sys.argv[1:]:
412        sys.argv.remove("--dxc-only")
413        packages=["dxc"]       
414
415    if "sdist" in sys.argv[1:]:
416        createSourceBundle(packages)
417    else:
418        if "install" not in sys.argv[1:]:
419            print "\nWARNING: Building without installing may mean your version won't pick up the location of the python packages."
420            time.sleep(0.3)   
421        print "\nChecking 'install.conf' configuration file to check you have reset all necessary parameters before beginning."
422        if open("../install.conf").read().find("PLEASE_REPLACE_ME")>-1:
423            print """Please manually edit the 'intall.conf' file and change all parameters that are currently defined as 'PLEASE_REPLACE_ME'.
424           
425This setup script creates (and can overwrite) some directories on your system so it is **CRUCIAL** that you ensure all
426the paths are set appropriately for your system!
427"""         
428        setupPackages(packages)
429       
Note: See TracBrowser for help on using the repository browser.