Changeset 562
- Timestamp:
- 26/01/16 21:57:31 (5 years ago)
- Location:
- hdldemo/trunk
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
hdldemo/trunk/hdldemo.py
r559 r562 6 6 -t: run a test; 7 7 -f <file name>: examine file, print path to replacement if this file is obsolete, print path to sibling files. 8 -id <tracking id>: examine handle record of tracking id. 8 9 """ 9 10 ## see https://www.handle.net/proxy_servlet.html for info on restfull API … … 11 12 import string, collections, os 12 13 import ncq3 14 import xml 15 from xml.dom import minidom 16 13 17 try: 14 18 import urllib … … 31 35 raise 32 36 33 34 37 class ghandle(object): 35 def __init__(self,url): 38 htmpl = 'http://hdl.handle.net/api/handles/%s' 39 dh = dummyHandles() 40 def __init__(self,hdl,url=None): 41 42 if hdl[:5] == 'xxxxx': 43 self.msg = self.dh.hh[hdl] 44 else: 45 if url == None: 46 thisid = string.replace(hdl,'hdl:999999', '10876.test' ) 47 url = self.htmpl % thisid 48 self.fetch(url) 49 50 def fetch(self,url): 51 print url 52 36 53 try: 37 54 fh = request.urlopen( url ) … … 47 64 self.msg = msg 48 65 66 class GHandles(object): 67 def __init__(self): 68 self.hh = {} 69 def __call__(self,hdl): 70 if hdl not in self.hh: 71 self.hh[hdl] = ghandle(hdl) 72 return self.hh[hdl] 73 74 ghandles = GHandles() 75 76 class pyhandle(object): 77 def __init__(self,id): 78 self.REC_id = id 79 self.REC_got = False 80 81 def get(self): 82 if self.REC_got: 83 return 84 85 g = ghandles( self.REC_id ) 86 for r in g.msg['values']: 87 if str(r['type']) in ['replaces', 'replacedBy', 'isReplacedBy','parent']: 88 self.__dict__[r['type']] = pyhandle( r['data']['value'] ) 89 90 elif r['type'] == 'children': 91 cl = eval( r['data']['value'] ) 92 self.__dict__['children'] = [pyhandle(c) for c in cl] 93 94 else: 95 self.__dict__[r['type']] = r['data']['value'] 96 97 def getSiblings(self): 98 if self.aggregation_level != 'file': 99 print 'No known siblings .....' 100 return 101 102 if 'parent' not in self.__dict__: 103 print 'No parent' 104 return 105 106 self.parent.get() 107 self.siblings = [] 108 for c in self.parent.children: 109 if c.REC_id != self.REC_id: 110 self.siblings.append( c ) 111 49 112 class main(object): 50 113 knownargs0 = ['-h','-v','-t'] 51 knownargs1 = ['-f' ]114 knownargs1 = ['-f','-id'] 52 115 def __init__(self, args): 53 116 self.htmpl = 'http://hdl.handle.net/api/handles/%s' … … 72 135 self.dumpF(fn) 73 136 74 def dumpF(self,fn): 75 assert os.path.isfile( fn ), 'File %s not found' % fn 76 f = ncHead( fn ) 77 thisid = string.replace(f.ga['tracking_id'], 'hdl:999999', '10876.test' ) 137 if '-id' in self.d: 138 id = self.d['-id'] 139 self.dumpF('',id=id) 140 141 def dumpF(self,fn, id=None): 142 if id == None: 143 assert os.path.isfile( fn ), 'File %s not found' % fn 144 f = ncHead( fn ) 145 thisid = string.replace(f.ga['tracking_id'], 'hdl:999999', '10876.test' ) 146 else: 147 thisid = string.replace(id,'hdl:999999', '10876.test' ) 148 78 149 url = self.htmpl % thisid 79 g = ghandle( url)150 g = ghandle( thisid ) 80 151 self.p = phandle( g.msg ) 81 gp = ghandle( self.htmpl %self.p.d['parent']['value'] )152 gp = ghandle( self.p.d['parent']['value'] ) 82 153 self.pp = phandle( gp.msg ) 83 if 'replaced_by' in self.p.d: 154 isReplaced = 'replaced_by' in self.p.d 155 if isReplaced: 84 156 print '******* OBSOLETE FILE ******' 85 pr = phandle( ghandle( self.htmpl % self.p.d['replaced_by']['value'] ).msg ) 86 print 'REPLACED BY: ',pr.d['URL']['value'] 157 pr = self.p 158 while isReplaced: 159 pr = phandle( ghandle( pr.d['replaced_by']['value'] ).msg ) 160 isReplaced = 'replaced_by' in pr.d 161 print 'REPLACED BY: ',pr.d['URL']['value'] 87 162 else: 88 163 print 'File is current' … … 95 170 thisfound = True 96 171 else: 97 gc = ghandle( self.htmpl %c )172 gc = ghandle( c ) 98 173 pc = phandle( gc.msg ) 99 if 'replaced_by' in pc.d: 100 pcr = phandle( ghandle( self.htmpl % pc.d['replaced_by']['value'] ).msg ) 101 print '>>>', pcr.d['URL']['value'] 174 isReplaced = 'replaced_by' in pc.d 175 rsq = [pc,] 176 if isReplaced: 177 while isReplaced: 178 pc = phandle( ghandle( pc.d['replaced_by']['value'] ).msg ) 179 isReplaced = 'replaced_by' in pc.d 180 print '>>>', pc.d['URL']['value'] 102 181 elif 'URL' in pc.d: 103 182 print pc.d['URL']['value'] … … 108 187 f = ncHead( fn1 ) 109 188 url = self.htmpl % f.ga['tracking_id'] 110 url = string.replace( url, 'hdl:999999', '10876.test' ) 111 print url 112 g = ghandle( url ) 189 hdl = string.replace( f.ga['tracking_id'], 'hdl:999999', '10876.test' ) 190 g = ghandle( hdl ) 113 191 self.p = phandle( g.msg ) 114 192 expected= ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'parent'] … … 122 200 print ('PARSING PARENT ..... ' ) 123 201 print ( self.htmpl % self.p.d['parent']['value'] ) 124 g = ghandle( self. htmpl % self.p.d['parent']['value'] )202 g = ghandle( self.p.d['parent']['value'] ) 125 203 self.pp = phandle( g.msg ) 126 204 for k in self.pp.d.keys(): -
hdldemo/trunk/testdata.py
r558 r562 2 2 Example page: https://handle8.dkrz.de/landingpagedemo/10876.test/ca9e9abd-e66e-413e-ab29-6c26fe00b859 3 3 """ 4 5 import uuid, collections, string, os 6 7 class dhdl(object): 8 def __init__(self,value=None): 9 if value == None: 10 self.value = 'xxxxx/%s' % str( uuid.uuid1() ) 11 else: 12 self.value = value 4 13 5 14 parentHandle1 = '10876.test/49634b69-6662-4a52-9175-45f296dc9578' … … 9 18 fn1 = 'novar_fx_dummy_historical_r3i1p1.nc' 10 19 20 expectedC = ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'parent'] 21 expectedP = ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'children', 'tracking_id'] 22 23 class map(object): 24 def __init__(self): 25 self.f = {} 26 self.t = {} 27 def add(self,a,b): 28 self.t[a] = b 29 self.f[b] = a 30 31 class dummyHandles(object): 32 ## 33 ## current data model does not support change in file name 34 ## 35 f1 = { 'ds1':( (('f1','aaaa'),('f2','bbbb')), \ 36 (('f1','aaaa'),('f2','bbbc')), \ 37 (('f1','aaaa'),('f3','bbb4')), \ 38 (('f1b','aaac'),('f3','bbb4')) ) } 39 mp = map() 40 for a,b in [('bbbb','bbbc'), ('bbbc','bbb4'),('aaaa','aaac') ]: 41 mp.add(a,b) 42 43 def __init__(self): 44 self.hh = {} 45 self.uds = collections.defaultdict( dhdl ) 46 self.uf = collections.defaultdict( dhdl ) 47 self.loadIds() 48 self.genDummy() 49 self.saveIds() 50 51 def loadIds(self): 52 if not os.path.isfile( 'cachedDummyHandles.txt' ): 53 return 54 55 oo = open('cachedDummyHandles.txt', 'r') 56 for l in oo.readlines(): 57 bits = string.split( string.strip( l ), '\t' ) 58 if bits[0] == 'f': 59 self.uf[bits[1]] = dhdl(value=bits[2]) 60 else: 61 self.uds[(bits[1], int(bits[2]) )] = dhdl(value=bits[3]) 62 oo.close() 63 64 def saveIds(self): 65 oo = open('cachedDummyHandles.txt', 'w') 66 for k in self.uds: 67 oo.write( 't\t%s\t%s\t%s\n' % (k[0],k[1],self.uds[k].value ) ) 68 for k in self.uf: 69 oo.write( 'f\t%s\t%s\n' % (k,self.uf[k].value ) ) 70 oo.close() 71 72 def genDummy(self): 73 dv = {} 74 ddv = {'aggregation_level':'dataset'} 75 ddf = {'aggregation_level':'file'} 76 self.dsd = {} 77 for ds in self.f1.keys(): 78 kds=0 79 for t in self.f1[ds]: 80 values = [] 81 kds += 1 82 cl = [] 83 for fn, cs in t: 84 u = self.uf[cs] 85 cl.append( u.value ) 86 if u.value not in self.hh: 87 extr = [] 88 ddf['tracking_id'] = u.value 89 ddf['parent'] = self.uds[(ds,kds)].value 90 if cs in self.mp.t: 91 ddf['replaced_by'] = self.uf[ self.mp.t[cs] ].value 92 extr.append( 'replaced_by' ) 93 print '>>> %s replaced_by %s' % (cs,ddf['replaced_by']) 94 if cs in self.mp.f: 95 ddf['replaces'] = self.uf[ self.mp.f[cs] ].value 96 extr.append( 'replaces' ) 97 print '>>> %s replaces %s' % (cs,ddf['replaces']) 98 thisV = [] 99 kk = 0 100 for k in expectedC + extr: 101 kk += 1 102 thisV.append( {'index':kk, 'type':k, 'data':{'format':'string', 'value':ddf.get(k,'dummy')} } ) 103 self.hh[u.value] = {"responseCode":1, "handle":u.value, "values":thisV} 104 dv[('children',kds)] = str(cl) 105 dv[('tracking_id',kds)] = self.uds[(ds,kds)].value 106 107 kk = 0 108 for k in expectedP: 109 kk += 1 110 values.append( {'index':kk, 'type':k, 'data':{'format':'string', 'value':dv.get((k,kds),ddv.get(k,'dummy'))} } ) 111 112 if kds < 4: 113 values.append( {'index':kk, 'type':'isReplacedBy', 'data':{'format':'string', 'value':self.uds[(ds,kds+1)].value} } ) 114 if kds > 1: 115 values.append( {'index':kk, 'type':'replaces', 'data':{'format':'string', 'value':self.uds[(ds,kds-1)].value} } ) 116 self.dsd[(ds,kds)] = {"responseCode":1,"handle":self.uds[(ds,kds)].value, "values":values} 117 self.hh[self.dsd[(ds,kds)]["handle"]] = self.dsd[(ds,kds)] 118 119 120 11 121 12 122 hsamp2 = {"responseCode":1,"handle":"10876.test/f05ca8f4-f011-11e4-8220-5404a60d96b5","values":[{"index":1,"type":"URL","data":{"format":"string","value":"http://clipc-services.ceda.ac.uk/testdata/v1/novar_fx_dummy_historical_r2i1p1.nc"},"ttl":86400,"timestamp":"2015-06-09T12:46:07Z"},{"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":"0.NA/10876.TEST","index":200,"permissions":"011111110011"}},"ttl":86400,"timestamp":"2015-06-09T12:46:07Z"},{"index":2,"type":"10320/loc","data":{"format":"string","value":"<locations chooseby=\"locatt,weighted\">\n<location ctype=\"text/html\" weight=\"1\" href=\"https://handle8.dkrz.de/landingpagedemo/10876.test/f05ca8f4-f011-11e4-8220-5404a60d96b5\" />\n<location http_role=\"conneg\" weight=\"0\" href=\"http://clipc-services.ceda.ac.uk/testdata/v1/novar_fx_dummy_historical_r2i1p1.nc\" />\n</locations>"},"ttl":86400,"timestamp":"2015-06-10T16:17:03Z"},{"index":3,"type":"creation_date","data":{"format":"string","value":"2015-05-01"},"ttl":86400,"timestamp":"2015-06-10T11:57:02Z"},{"index":4,"type":"tracking_id","data":{"format":"string","value":"f05ca8f4-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-10T11:57:08Z"},{"index":5,"type":"replaced_by","data":{"format":"string","value":"10876.test/f05f7caa-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-10T11:57:14Z"},{"index":6,"type":"checksum","data":{"format":"string","value":"ad99a037946fc7e9eaa1d916c8682a11"},"ttl":86400,"timestamp":"2015-06-09T12:46:07Z"},{"index":7,"type":"parent","data":{"format":"string","value":"10876.test/49634b69-6662-4a52-9175-45f296dc9578"},"ttl":86400,"timestamp":"2015-06-10T11:57:21Z"},{"index":8,"type":"aggregation_level","data":{"format":"string","value":"file"},"ttl":86400,"timestamp":"2015-06-10T11:57:34Z"}]} 13 123 hsample = {"responseCode":1,"handle":"10876.test/f0600a58-f011-11e4-8220-5404a60d96b5","values":[{"index":1,"type":"URL","data":{"format":"string","value":"http://clipc-services.ceda.ac.uk/testdata/v2/novar_fx_dummy_historical_r3i1p1.nc"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":"0.NA/10876.TEST","index":200,"permissions":"011111110011"}},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":2,"type":"10320/loc","data":{"format":"string","value":"<locations chooseby=\"locatt,weighted\">\n<location ctype=\"text/html\" weight=\"1\" href=href=\"https://handle8.dkrz.de/landingpagedemo/10876.test/f0600a58-f011-11e4-8220-5404a60d96b5\" />\n<location http_role=\"conneg\" weight=\"0\" href=\"http://clipc-services.ceda.ac.uk/testdata/v2/novar_fx_dummy_historical_r3i1p1.nc\" />\n</locations>"},"ttl":86400,"timestamp":"2015-06-10T16:21:11Z"},{"index":3,"type":"creation_date","data":{"format":"string","value":"2015-05-01"},"ttl":86400,"timestamp":"2015-06-10T12:01:04Z"},{"index":4,"type":"replaces","data":{"format":"string","value":"10876.test/f05d326a-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":5,"type":"trackingID","data":{"format":"string","value":"f0600a58-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":6,"type":"checksum","data":{"format":"string","value":"e2e71fe45c106298dc15fbd08f9707e1"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":7,"type":"parent","data":{"format":"string","value":"10876.test/ca9e9abd-e66e-413e-ab29-6c26fe00b859"},"ttl":86400,"timestamp":"2015-06-10T12:01:10Z"},{"index":8,"type":"aggregation_level","data":{"format":"string","value":"file"},"ttl":86400,"timestamp":"2015-06-10T12:01:21Z"}]} 124 125 psampl1 = {"responseCode":1,"handle":"10876.test/49634b69-6662-4a52-9175-45f296dc9578","values":[{"index":1,"type":"URL","data":{"format":"string","value":"http://clipc-services.ceda.ac.uk/testdata/v1"},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":"0.NA/10876.TEST","index":200,"permissions":"011111110011"}},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":2,"type":"10320/loc","data":{"format":"string","value":"<locations chooseby=\"locatt,weighted\">\n<location ctype=\"text/html\" weight=\"1\" href=href=\"https://handle8.dkrz.de/landingpagedemo/10876.test/49634b69-6662-4a52-9175-45f296dc9578\" />\n<location http_role=\"conneg\" weight=\"0\" href=\"http://clipc-services.ceda.ac.uk/testdata/v1/\" />\n</locations>"},"ttl":86400,"timestamp":"2015-06-10T16:24:37Z"},{"index":3,"type":"creation_date","data":{"format":"string","value":"2015-05-01"},"ttl":86400,"timestamp":"2015-06-10T12:04:29Z"},{"index":4,"type":"tracking_id","data":{"format":"string","value":"none"},"ttl":86400,"timestamp":"2015-06-10T12:04:35Z"},{"index":5,"type":"isReplacedBy","data":{"format":"string","value":"10876.test/ca9e9abd-e66e-413e-ab29-6c26fe00b859"},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":6,"type":"checksum","data":{"format":"string","value":"todo"},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":7,"type":"children","data":{"format":"string","value":"[\"10876.test/f05e5f1e-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05d326a-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05ca8f4-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05c1876-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05dc00e-f011-11e4-8220-5404a60d96b5\"]"},"ttl":86400,"timestamp":"2015-06-10T12:04:46Z"},{"index":8,"type":"aggregation_level","data":{"format":"string","value":"dataset"},"ttl":86400,"timestamp":"2015-06-10T12:04:57Z"}]}
Note: See TracChangeset
for help on using the changeset viewer.