source: TI03-DataExtractor/trunk/setup.py @ 1109

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/trunk/setup.py@1109
Revision 1109, 16.5 KB checked in by astephen, 13 years ago (diff)

Stable-ish version with fully-ish working dxc client.

Line 
1#!/usr/bin/env python
2
3"""
4setup.py
5========
6
7Set up script for the Data Extractor package.
8
9The Data Extractor is a server package (dxs) that is exposed
10as a Web Service using python's ZSI SOAP Library.
11
12There is also a client package to primarily support the CGI client
13package (dxc).
14
15This installation includes both server and client and assumes you want
16to install both on your local machine. If you only wish to install either
17server or client use one of the following options:
18
19--dxc-only # only install the client package
20--dxs-only # only install the server package
21
22
23Usage:
24======
25
26    [<your_python>/bin/python] setup.py [options] 
27
28[options] can include the above as well as the usual
29
30build | install
31
32Note that using "--overwrite" will automatically overwrite
33existing directories regardless of whether
34
35"""
36
37from distutils.core import setup
38import sys, os, shutil, time, re
39from ConfigParser import ConfigParser
40version="0.2.1"
41overwrite="OFF"
42
43
44def createSourceBundle(packages=["dxs", "dxc"]):
45    """
46    Creates the packages required and includes the directories needed.
47    """
48    basePackage=packages[0][:2]
49    packMap={"s":"Server", "c":"Client"}
50       
51    for package in packages: 
52        if os.path.isfile("MANIFEST"): os.remove("MANIFEST")
53        pypackage="py"+package
54        packageName="Python Data Extractor %s" % packMap[pypackage[-1]]
55        print "\nBundling up source of python package: %s" % packageName
56   
57        setup(name=pypackage,
58            version=version,
59            description=packageName,
60            author="Ag Stephens",           
61            author_email="a.stephens@rl.ac.uk",
62            url="http://ndg.nerc.ac.uk/DataExtractor",
63            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
64This software may be distributed under the terms of the
65Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
66            packages=[pypackage])   
67           
68        print "Bundling up the source of the non-python stuff and python scripts for: %s" % packageName
69        tarfile="%s-%s.tar" % (package, version)
70        gzipfile=tarfile+".gz"
71        os.system("tar -cvf %s %s" % (tarfile, package))
72        os.system("gzip %s" % tarfile)
73       
74        if not os.path.isfile(gzipfile):
75            raise "Did not successfully package up non-python source for '%s'." % package
76           
77        print "Moving gzip file into distribution..."
78        os.rename(gzipfile, os.path.join("dist", gzipfile))
79
80        if package=="dxc":
81            print "Copying cgi directory..."
82            if os.path.isdir("dist/cgi"):
83                if os.path.isfile("dist/cgi/dxui"):
84                    os.remove("dist/cgi/dxui")
85                os.rmdir("dist/cgi")
86            shutil.copytree("cgi", "dist/cgi")
87           
88            print "Copying web_extras directory..."
89            if os.path.isdir("dist/web_extras"):
90                os.system("rm -fR dist/web_extras")
91            shutil.copytree("web_extras", "dist/web_extras")       
92           
93    print "Copying README.txt and install.conf files..."
94    shutil.copy("README.txt", "dist")
95    shutil.copy("install.sh", "dist")   
96    shutil.copy("install.conf", "dist") 
97    shutil.copy("install.conf", "dist/install.conf.orig")
98       
99    print "Tarring up everything in dist directory..."
100    tarAllFile="%s-%s-all.tar" % (basePackage, version)
101    os.system("cd dist ; tar -cvf %s *" % tarAllFile)
102       
103    if not os.path.isfile(os.path.join("dist", tarAllFile)):
104        raise "Did not successfully package up tar-all file '%s'." % tarAllFile
105       
106
107
108def setupPackages(packages=["dxs", "dxc"]):
109    """
110    Controls the entire setup and creates directories and
111    permissions where necessary to get the ancillary stuff working.
112    """   
113    testImportExternalPackages()
114   
115    packMap={"s":"Server", "c":"Client"}
116       
117    for package in packages: 
118        pypackage="py"+package
119        packageName="Python Data Extractor %s" % packMap[package[-1]]
120           
121        print "Installing the non-python stuff and python scripts for: %s" % packageName
122        unpackageDir="%s-%s" % (package, version)
123        tarfile="%s.tar" % unpackageDir
124        gzipfile=tarfile+".gz"
125        os.system("cd .. ; gunzip %s" % tarfile)       
126        os.system("cd .. ; tar -xvf %s" % tarfile)
127
128        packageDir=os.path.join("..", package)
129        if not os.path.isdir(packageDir):
130            raise "Did not successfully unpack the source files for %s." % package
131           
132        print "Reading configuration file to get %s local details." % package
133        config=ConfigParser()
134        config.read("../install.conf")
135 
136        print "Set the configuration variables that are used to generate others..."
137        for item in ["BASEDIR", "STATIC_WEBSERVER_LOCAL_PATH", "STATIC_WEBSERVER_URL_PATH",
138                     "WEB_EXTRAS_URL_PATH"]:
139            if item.lower() in config.options(package):
140                exec("%s=%s" % (item.upper(), config.get(package, item.lower())))
141                print "SET:", item
142 
143        print "\n\n"
144        configVarDict={}
145        for configVar in config.options(package):
146                value=config.get(package, configVar)
147                print "Setting: %s=%s" % (configVar.upper(), value)
148                exec("%s=%s" % (configVar.upper(), value))
149                configVarDict[configVar.upper()]=value
150
151        print "Now we have parsed the config file we can start copying and making the source directories."
152        print "Installing base package '%s' in: %s" % (package, BASEDIR)
153
154        if overwrite!="ON":
155            checkBeforeMakingDir(BASEDIR)
156           
157            """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % BASEDIR)
158            if destroy!="yes":
159                print "Exiting setup without completing."
160                sys.exit()
161           
162        if not os.path.isdir(BASEDIR):
163            try:
164                os.makedirs(BASEDIR)
165            except:
166                pass"""
167       
168        for item in os.listdir(packageDir):
169            sourceCopy=os.path.join(packageDir, item)
170            target=os.path.join(BASEDIR, item)
171            if os.path.exists(target): 
172                os.system("rm -fr %s" % target)
173           
174            if os.path.isfile(sourceCopy):
175                shutil.copy(sourceCopy, target)
176            elif os.path.isdir(sourceCopy):
177                #if os.path.split(sourceCopy)[-1]=="logs":
178                #    print "Not overwriting logs directory."
179                #    if not os.path.isdir(
180                #else:
181                    shutil.copytree(sourceCopy, target)
182       
183        print "Installed non-python package and scripts under:", BASEDIR
184
185        if "OUTPUT_DIR" in configVarDict.keys():
186            if overwrite!="ON":
187                checkBeforeMakingDir(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP)           
188               
189                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % OUTPUT_DIR)
190                if destroy!="yes":
191                    print "Exiting setup without completing."
192                    sys.exit()   
193            try:
194                os.system("rm -fR %s" % OUTPUT_DIR)
195                os.makedirs(OUTPUT_DIR, OUTPUT_DIR_PERMISSION)
196                #os.chmod(OUTPUT_DIR, OUTPUT_DIR_PERMISSION)
197                os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
198            except:
199                raise "Cannot create directory: %s" % OUTPUT_DIR"""
200            else:
201                makeDirAndPerm(OUTPUT_DIR, OUTPUT_DIR_PERMISSION, OUTPUT_FILE_USER, OUTPUT_FILE_GROUP) 
202
203        for dirToMake in ["OUTPUT_DIR_LOCAL_PATH"]:
204         
205          if dirToMake in configVarDict.keys():
206            dirToMake=eval(configVarDict[dirToMake])     
207            if overwrite!="ON":
208                checkBeforeMakingDir(OUTPUT_DIR_LOCAL_PATH)             
209                   
210           
211                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % dirToMake)
212                if destroy!="yes":
213                    print "Exiting setup without completing."
214                    sys.exit()   
215            try:
216                os.system("rm -fR %s" % dirToMake)
217                os.makedirs(dirToMake, 0755)
218                #os.chmod(dirToMake, 0755)
219                #os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
220            except:
221                raise "Cannot create directory: %s" % dirToMake"""
222               
223        if "CGI_SCRIPT_LOCAL_PATH" in configVarDict.keys():
224            """if overwrite!="ON":
225                destroy=raw_input("Type 'yes' to remove existing script before re-installing: '%s' :" % CGI_SCRIPT_URL_PATH)
226                if destroy!="yes":
227                    print "Exiting setup without completing."
228                    sys.exit()"""   
229            try:
230                shutil.copy("../cgi/dxui", CGI_SCRIPT_LOCAL_PATH)
231                os.chmod(CGI_SCRIPT_LOCAL_PATH, 0755)
232                thisPython=sys.modules["os"].__file__.split("lib")[0]+"bin/python"
233                print thisPython
234                os.system("perl -p -i -w -e 's:<YOUR_PYTHON_HERE>:%s:g;' %s*" % (thisPython, CGI_SCRIPT_LOCAL_PATH))
235                #os.system("perl -p -i -w -e 's!BASEDIR=\".*?\"!BASEDIR=\"%s\"!g;' %s*" % ("/home/as56/abstractWebService/test_installs/20060328/pydxc-0.1/pydxc", CGI_SCRIPT_LOCAL_PATH))
236                print "\nCopied CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
237            except:
238                raise "Cannot move CGI script to: %s" % CGI_SCRIPT_LOCAL_PATH
239
240        if "WEB_EXTRAS_LOCAL_PATH" in configVarDict.keys():
241            if overwrite!="ON":
242                checkBeforeMakingDir(WEB_EXTRAS_LOCAL_PATH, 0755)                   
243                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % WEB_EXTRAS_LOCAL_PATH)
244                if destroy!="yes":
245                    print "Exiting setup without completing."
246                    sys.exit()   
247            try:
248                os.system("rm -fR %s" % WEB_EXTRAS_LOCAL_PATH)
249                os.makedirs(WEB_EXTRAS_LOCAL_PATH, 0755)
250                #os.chmod(OUTPUT_DIR, OUTPUT_DIR_PERMISSION)
251                #os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
252            except:
253                raise "Cannot create directory: %s" % WEB_EXTRAS_LOCAL_PATH  """               
254
255            webExtrasDir="../web_extras"
256            for item in os.listdir(webExtrasDir):
257                sourceCopy=os.path.join(webExtrasDir, item)
258                target=os.path.join(WEB_EXTRAS_LOCAL_PATH, item)
259                if os.path.exists(target): 
260                    os.system("rm -fR %s" % target)
261           
262                if os.path.isfile(sourceCopy):
263                    shutil.copy(sourceCopy, target)
264                elif os.path.isdir(sourceCopy):
265                    shutil.copytree(sourceCopy, target)
266       
267            print "Installed web extras components under:", WEB_EXTRAS_LOCAL_PATH
268
269        if "REQUEST_XML_DIR_LOCAL_PATH" in configVarDict.keys():
270            if overwrite!="ON":
271                checkBeforeMakingDir(REQUEST_XML_DIR_LOCAL_PATH, 0777)             
272           
273                """destroy=raw_input("Type 'yes' to remove existing directory before re-installing: '%s' :" % REQUEST_XML_DIR_LOCAL_PATH)
274                if destroy!="yes":
275                    print "Exiting setup without completing."
276                    sys.exit()   
277            try:
278                os.system("rm -fR %s" % REQUEST_XML_DIR_LOCAL_PATH)
279                os.makedirs(REQUEST_XML_DIR_LOCAL_PATH)
280                print "Made Request XML directory: %s" % REQUEST_XML_DIR_LOCAL_PATH
281                os.chmod(REQUEST_XML_DIR_LOCAL_PATH, 0777)
282                #os.system("chown -Rf %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, OUTPUT_DIR))
283            except:
284                raise "Cannot create directory: %s" % REQUEST_XML_DIR_LOCAL_PATH """
285       
286        configMap={"s":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "serverConfig.py"), 
287                   "c":os.path.join("..", "%s-%s" % (pypackage, version), pypackage, "clientConfig.py")}
288        print "Renaming configuration variables in config file:"
289       
290        configFileName=configMap[package[-1]]
291        configFile=open(configFileName)
292        configLines=configFile.readlines()
293        configFile.close()
294       
295        shutil.copy(configFileName, configFileName+".bak")
296       
297        newLines=[]
298        for line in configLines:
299            matched=None
300            for confVar, value in configVarDict.items():
301               
302                if re.match(r"\s*%s\s*=" % confVar, line):
303                    newLine="%s=%s\n" % (confVar, value)
304                    print "CHANGING:   %sTO:         %s" % (line, newLine)
305                    newLines.append(newLine)
306                    matched=1
307                    break
308                   
309            if matched==None:
310                newLines.append(line)     
311
312        print len(configLines), len(newLines)
313        configFile=open(configFileName, "w")
314        configFile.write("".join(newLines))
315        configFile.close()
316        print "Re-written config file: %s\n" % (configFileName)
317
318        if package=="dxs":
319            print "Re-naming dataset paths in inputDatasets.xml file...\n"
320            inputPath="DATASET_PATH_TO_CHANGE"
321            outputPath=os.path.join(BASEDIR, "testdata")
322            inputDatasetsFile=os.path.join(BASEDIR, "datasets", "inputDatasets.xml")
323            os.system("perl -p -i -w -e 's!%s!%s!g;' %s*" % (inputPath, outputPath, inputDatasetsFile))
324            print "Done.\n"
325
326        print "Installing python package: %s" % packageName
327
328        os.chdir(os.path.join("..", "%s-%s" % (pypackage, version)))
329        setup(name=pypackage,
330            version=version,
331            description=packageName,
332            author="Ag Stephens",           
333            author_email="a.stephens@rl.ac.uk",
334            url="http://ndg.nerc.ac.uk/DataExtractor",
335            license="""Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
336This software may be distributed under the terms of the
337Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QP""",
338            packages=[pypackage])       
339   
340    print "\nIt looks like setup has completed successfully!\n"
341    print """You can start the server Web Service by typing:
342    python %s/bin/DXWSInterface.py
343
344You should then be able to view the client package via a browser at:
345    %s
346   
347Have fun!....and of course send any bugs to <a.stephens@rl.ac.uk>
348
349""" % (BASEDIR, CGI_SCRIPT_URL_PATH)
350   
351   
352def testImportExternalPackages():
353    """
354    Tests that external packages can be imported.
355    """
356    print "Testing import of required cdms (CDAT) package...",
357    try:
358        import cdms         
359        print "OK!"
360    except:
361        print """\n\nCannot import cdms package. Please try the one or more of the following to fix this:
362
3631. Set your PYTHONPATH environment variable to pick up the correct directory containing the cdms package.
3642. Retry using the correct python installation such as "<my_python>/bin/python setup.py"
3653. Install CDAT (cdms)."""
366        sys.exit()
367
368    print "Testing import required ZSI package...",
369    try:
370        import ZSI
371        print "OK!"
372    except:
373        print """\n\nCannot import ZSI package. Please try one or more of the following to fix this:
374
3751. Set your PYTHONPATH environment variable to pick up the correct directory containing the ZSI package.
3762. Install ZSI."""
377        sys.exit()
378 
379 
380def checkBeforeMakingDir(dirname, permissions=0755, owner=None, group=None):
381    """
382   Checks with the user before deleting and re-creating directories.
383   """
384    if os.path.isdir(dirname):
385        print "\nThis package needs uses the following directory which already exists:\n",dirname
386        cleanOut=raw_input("Type 'yes' to clean out ALL existing contents of this directory or press return to only overwrite files being added:")
387        if cleanOut=="yes":         
388            removeDirWithCheck(dirname)
389           
390    sep=os.path.sep
391    pathParts=dirname.split(sep)
392    cpath=""
393    for part in pathParts:
394        cpath=cpath+sep+part
395        makeDirAndPerm(dirname, permissions=permissions, owner=owner, group=group)
396    print "Directory is ready: ", dirname       
397
398
399def removeDirWithCheck(dirname):
400    """
401    Removes dir but asks users permission.
402    """
403    if os.path.isdir(dirname):
404        deleteIt=raw_input("Type 'yes' to confirm you REALLY want to delete the following directory '%s':" % dirname)
405        if deleteIt=="yes":
406            os.system("rm -fR %s" % dirname)
407
408
409def makeDirAndPerm(dirname, permissions=0755, owner=None, group=None):
410    """
411    Tries to make a directory with appropriate permissions, owner etc.
412    """
413    if not os.path.isdir(dirname):
414        parentDir=os.path.split(dirname)[0]
415        if not os.path.isdir(parentDir):
416            makeDirAndPerm(parentDir)
417        os.mkdir(dirname)
418        os.chmod(dirname, permissions)
419        os.system("chown %s %s" % (owner, dirname))
420        os.system("chgrp %s %s" % (group, dirname))
421   
422
423if __name__=="__main__":
424
425    packages=["dxs", "dxc"]       
426
427    if "--overwrite" in sys.argv[1:]:
428        print "Setting overwrite to ON"
429        overwrite="ON"
430        sys.argv.remove("--overwrite")
431       
432    if "--dxs-only" in sys.argv[1:]:
433        sys.argv.remove("--dxs-only")
434        packages=["dxs"]
435    elif "--dxc-only" in sys.argv[1:]:
436        sys.argv.remove("--dxc-only")
437        packages=["dxc"]       
438
439    if "sdist" in sys.argv[1:]:
440        createSourceBundle(packages)
441    else:
442        if "install" not in sys.argv[1:]:
443            print "\nWARNING: Building without installing may mean your version won't pick up the location of the python packages."
444            time.sleep(0.3)   
445        print "\nChecking 'install.conf' configuration file to check you have reset all necessary parameters before beginning."
446        if open("../install.conf").read().find("PLEASE_REPLACE_ME")>-1:
447            print """Please manually edit the 'intall.conf' file and change all parameters that are currently defined as 'PLEASE_REPLACE_ME'.
448           
449This setup script creates (and can overwrite) some directories on your system so it is **CRUCIAL** that you ensure all
450the paths are set appropriately for your system!
451"""         
452        setupPackages(packages)
453       
Note: See TracBrowser for help on using the repository browser.