source: TI12-security/trunk/esg_wget_script/esg-recursive-download.sh @ 7251

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI12-security/trunk/esg_wget_script/esg-recursive-download.sh@7251
Revision 7251, 4.5 KB checked in by pjkersha, 9 years ago (diff)

WGet scripts for ESG data download

Line 
1#
2# ESG Download script wraps wget call with settings for ESG Security
3#
4# @author P J Kershaw 28/07/2010
5#
6# @copyright: (C) 2010 STFC
7#
8# @license: BSD - See top-level LICENCE file for licence details
9#
10# $Id$
11cmdname=$(basename $0)
12cmdline_opt=`getopt -o hO: --long help,output-document:,certificate:,private-key:,ca-directory:,save-cookies:: -n "$cmdname" -- "$@"`
13
14defaultCaDir=$HOME/.globus/certificates/esg
15esgDotDir=$HOME/.esg
16defaultCookieFile=$esgDotDir/cookies.txt
17usage="Usage: $cmdname <data download URI> <options ...>\n
18\n
19Script for Earth System Grid recursive data download.\n\n
20
21   Options\n
22       -h | --help\t\t\t\tDisplays usage\n
23       -O | --output-document\t<filepath>\tLocation of output file (defaults to appropriate file name based on requested URI\n
24       --certificate\t<certificate file>\tSSL certificate to authenticate with (PEM format).\n
25       \t\t\t\t\tDefaults to X509_USER_PROXY or X509_USER_CERT if set.  If using X509_USER_PROXY,\n
26       \t\t\t\t\tit must point to a file containing the concatenated certificate and private\n
27       \t\t\t\t\tkey files.\n
28       --private-key\t<private key file>\tfile containing private key for SSL authentication (PEM format)\n
29       \t\t\t\t\tDefaults to X509_USER_PROXY or X509_USER_KEY if set.\n
30       --ca-directory\t<directory path>\tDirectory containing the trusted CA (Certificate Authority) certificates used\n
31       \t\t\t\t\tto verify the identity of the server (defaults to \n
32       \t\t\t\t\t$defaultCaDir or may be set from the X509_CERT_DIR\n
33       \t\t\t\t\tenvironment variable).  The CA files can be obtained by a call\n
34       \t\t\t\t\tto MyProxy logon saving 'trust roots' to the selected CA directory.\n
35       --save-cookies\t<cookie file>\t\tSave cookies to this file.  The default location is $defaultCookieFile
36"
37
38if [ $? != 0 ] ; then
39    echo -e $usage >&2 ;
40    exit 1 ;
41fi
42
43eval set -- "$cmdline_opt"
44
45while true ; do
46    case "$1" in
47        -h|--help) echo -e $usage ; exit 0 ;;
48        --certificate) certFile=$2 ; shift 2 ;;
49        --private-key) privateKeyFile=$2 ; shift 2 ;;
50        --ca-directory) caDir=$2 ; shift 2 ;;
51        -O|--output-document) outputFile=$2 ; shift 2 ;;
52        --save-cookies) cookieFile=$2 ; shift 2 ;;
53        --) uri=$2 ; shift 1 ; break ;;
54        *) echo "Error parsing command line" ; exit 1 ;;
55#        *) echo "uri opt: $1" ; uri=$1 ; shift 1 ; break ;;
56    esac
57done
58
59if [ -z $uri ]; then
60    echo "Error: missing download URI." >&2 ;
61    echo -e $usage >&2 ;
62    exit 1 ;
63fi
64   
65# Set-up trust root
66if [ -z $caDir ]; then 
67    if [ ${X509_CERT_DIR} ]; then
68        caDir=${X509_CERT_DIR}
69    elif [ "$username" = "root" ]; then
70        caDir=/etc/grid-security/certificates
71    else
72        caDir=${HOME}/.globus/certificates/esg
73    fi
74fi
75
76# Set-up client certificate and private key
77if [ -z $certFile ]; then
78    if [ ${X509_USER_PROXY} ]; then
79        # This environment variable setting means both cert and key are
80        # concatenated together in the same file
81        certFile=${X509_USER_PROXY}
82        privateKeyFile=${X509_USER_PROXY}
83       
84    elif [ ${X509_USER_CERT} ]; then
85        certFile=${X509_USER_CERT}   
86    fi
87   
88    # No check for cert not set because this is a valid condition if the data
89    # requested is not secured.
90fi
91
92if [ -z $privateKeyFile ]; then
93    if [ ${X509_USER_KEY} ]; then
94        privateKeyFile=${X509_USER_KEY}
95    fi
96   
97    # No check for key not set because this is a valid condition if the data
98    # requested is not secured
99fi
100
101# Set-up the cookie file path
102if [ -z $cookieFile ]; then
103    cookieFile=$defaultCookieFile
104    if [ ! -d $esgDotDir ]; then
105        mkdir $esgDotDir ;
106    fi
107fi
108
109if [ $outputFile ]; then
110    outputFileSetting=--output-document=$outputFile
111else
112    outputFileSetting=
113fi
114
115
116# Two stage wget call to workaround recursive mode (-r) not working with
117# redirects:
118#
119# 1) Authenticate - this involves redirects to and back from SSL endpoint
120#    submitting a certificate and saving a session cookie. 
121wget --spider \
122 --ca-directory=$caDir \
123 --certificate=$certFile \
124 --private-key=$privateKeyFile \
125 --keep-session-cookies \
126 --save-cookies=$cookieFile \
127 --cookies=on \
128 $uri
129 
130# 2) Call again with session cookie to allow for the recursive case - with a
131#    cookie set, no redirects are invoked and so in this case recursive download
132#    will succeed.  If the request was for a single file, step 1 would have
133#    completed the job.  -nc is set here in step 2) to avoid re-downloading the
134#    same file
135wget -r -nc \
136 --load-cookies=$cookieFile $outputFileSetting \
137 $uri
Note: See TracBrowser for help on using the repository browser.