Changeset 562

Timestamp:

26/01/16 21:57:31 (3 years ago)

Author:

mjuckes

Message:

improved class structures

Location:

hdldemo/trunk

Files:

• hdldemo.py (modified) (8 diffs)
• testdata.py (modified) (2 diffs)

hdldemo/trunk/hdldemo.py

@@ -6 +6 @@
  -t: run a test;
  -f <file name>: examine file, print path to replacement if this file is obsolete, print path to sibling files.
+ -id <tracking id>: examine handle record of tracking id.
 """
 ## see https://www.handle.net/proxy_servlet.html for info on restfull API
@@ -11 +12 @@
 import string, collections, os
 import ncq3
+import xml
+from xml.dom import minidom
+
 try:
  import urllib
@@ -31 +35 @@
       raise
 
-
 class ghandle(object):
-  def __init__(self,url):
+  htmpl = 'http://hdl.handle.net/api/handles/%s'
+  dh = dummyHandles()
+  def __init__(self,hdl,url=None):
+
+     if hdl[:5] == 'xxxxx':
+       self.msg = self.dh.hh[hdl]
+     else:
+       if url == None:
+         thisid = string.replace(hdl,'hdl:999999', '10876.test' )
+         url = self.htmpl % thisid
+       self.fetch(url)
+
+  def fetch(self,url):
+     print url
+
      try:
        fh = request.urlopen( url )
@@ -47 +64 @@
      self.msg = msg
 
+class GHandles(object):
+  def __init__(self):
+    self.hh = {}
+  def __call__(self,hdl):
+    if hdl not in self.hh:
+      self.hh[hdl] = ghandle(hdl)
+    return self.hh[hdl]
+
+ghandles = GHandles()
+
+class pyhandle(object):
+  def __init__(self,id):
+    self.REC_id = id
+    self.REC_got = False
+
+  def get(self):
+    if self.REC_got:
+      return
+
+    g = ghandles( self.REC_id )
+    for r in g.msg['values']:
+        if str(r['type']) in ['replaces', 'replacedBy', 'isReplacedBy','parent']:
+          self.__dict__[r['type']] = pyhandle( r['data']['value'] )
+
+        elif r['type']  == 'children':
+          cl = eval( r['data']['value'] )
+          self.__dict__['children'] = [pyhandle(c) for c in cl]
+
+        else:
+          self.__dict__[r['type']] = r['data']['value']
+
+  def getSiblings(self):
+     if self.aggregation_level != 'file':
+       print 'No known siblings .....'
+       return
+
+     if 'parent' not in self.__dict__:
+       print 'No parent'
+       return
+
+     self.parent.get()
+     self.siblings = []
+     for c in self.parent.children:
+       if c.REC_id != self.REC_id:
+         self.siblings.append( c )
+
 class main(object):
   knownargs0 = ['-h','-v','-t']
-  knownargs1 = ['-f']
+  knownargs1 = ['-f','-id']
   def __init__(self, args):
     self.htmpl = 'http://hdl.handle.net/api/handles/%s'
@@ -72 +135 @@
       self.dumpF(fn)
 
-  def dumpF(self,fn):
-    assert os.path.isfile( fn ), 'File %s not found' % fn
-    f = ncHead( fn )
-    thisid = string.replace(f.ga['tracking_id'], 'hdl:999999', '10876.test' )
+    if '-id' in self.d:
+      id = self.d['-id']
+      self.dumpF('',id=id)
+
+  def dumpF(self,fn, id=None):
+    if id == None:
+      assert os.path.isfile( fn ), 'File %s not found' % fn
+      f = ncHead( fn )
+      thisid = string.replace(f.ga['tracking_id'], 'hdl:999999', '10876.test' )
+    else:
+      thisid = string.replace(id,'hdl:999999', '10876.test' )
+
     url = self.htmpl % thisid
-    g = ghandle( url )
+    g = ghandle( thisid )
     self.p = phandle(  g.msg )
-    gp = ghandle( self.htmpl % self.p.d['parent']['value'] )
+    gp = ghandle(  self.p.d['parent']['value'] )
     self.pp = phandle(  gp.msg )
-    if 'replaced_by' in self.p.d:
+    isReplaced = 'replaced_by' in self.p.d
+    if isReplaced:
       print '******* OBSOLETE FILE ******'
-      pr = phandle(  ghandle( self.htmpl % self.p.d['replaced_by']['value'] ).msg )
-      print 'REPLACED BY: ',pr.d['URL']['value']
+      pr = self.p
+      while isReplaced:
+        pr = phandle(  ghandle( pr.d['replaced_by']['value'] ).msg )
+        isReplaced = 'replaced_by' in pr.d
+        print 'REPLACED BY: ',pr.d['URL']['value']
     else:
       print 'File is current'
@@ -95 +170 @@
         thisfound = True
       else:
-        gc = ghandle( self.htmpl %  c )
+        gc = ghandle(  c )
         pc = phandle( gc.msg )
-        if 'replaced_by' in pc.d:
-          pcr = phandle(  ghandle( self.htmpl % pc.d['replaced_by']['value'] ).msg )
-          print '>>>', pcr.d['URL']['value']
+        isReplaced = 'replaced_by' in pc.d
+        rsq = [pc,]
+        if isReplaced:
+          while isReplaced:
+            pc = phandle(  ghandle( pc.d['replaced_by']['value'] ).msg )
+            isReplaced = 'replaced_by' in pc.d
+            print '>>>', pc.d['URL']['value']
         elif 'URL' in pc.d:
           print pc.d['URL']['value']
@@ -108 +187 @@
     f = ncHead( fn1 )
     url = self.htmpl % f.ga['tracking_id']
-    url = string.replace( url, 'hdl:999999', '10876.test' )
-    print url
-    g = ghandle( url )
+    hdl = string.replace( f.ga['tracking_id'], 'hdl:999999', '10876.test' )
+    g = ghandle( hdl )
     self.p = phandle(  g.msg )
     expected= ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'parent']
@@ -122 +200 @@
     print ('PARSING PARENT ..... ' )
     print ( self.htmpl % self.p.d['parent']['value'] )
-    g = ghandle( self.htmpl % self.p.d['parent']['value'] )
+    g = ghandle( self.p.d['parent']['value'] )
     self.pp = phandle(  g.msg )
     for k in self.pp.d.keys():

hdldemo/trunk/testdata.py

@@ -2 +2 @@
 Example page: https://handle8.dkrz.de/landingpagedemo/10876.test/ca9e9abd-e66e-413e-ab29-6c26fe00b859
 """
+
+import uuid, collections, string, os
+
+class dhdl(object):
+  def __init__(self,value=None):
+    if value == None:
+      self.value = 'xxxxx/%s' % str( uuid.uuid1() )
+    else:
+      self.value = value
 
 parentHandle1 = '10876.test/49634b69-6662-4a52-9175-45f296dc9578'
@@ -9 +18 @@
 fn1 = 'novar_fx_dummy_historical_r3i1p1.nc'
 
+expectedC = ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'parent']
+expectedP = ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'children', 'tracking_id']
+
+class map(object):
+  def __init__(self):
+    self.f = {}
+    self.t = {}
+  def add(self,a,b):
+    self.t[a] = b
+    self.f[b] = a
+
+class dummyHandles(object):
+  ##
+  ## current data model does not support change in file name
+  ##
+  f1 = { 'ds1':( (('f1','aaaa'),('f2','bbbb')), \
+                 (('f1','aaaa'),('f2','bbbc')), \
+                 (('f1','aaaa'),('f3','bbb4')), \
+                 (('f1b','aaac'),('f3','bbb4')) ) }
+  mp = map()
+  for a,b in [('bbbb','bbbc'), ('bbbc','bbb4'),('aaaa','aaac') ]:
+    mp.add(a,b)
+
+  def __init__(self):
+    self.hh = {}
+    self.uds = collections.defaultdict( dhdl )
+    self.uf = collections.defaultdict( dhdl )
+    self.loadIds()
+    self.genDummy()
+    self.saveIds()
+
+  def loadIds(self):
+    if not os.path.isfile( 'cachedDummyHandles.txt' ):
+      return
+
+    oo = open('cachedDummyHandles.txt', 'r')
+    for l in oo.readlines():
+      bits = string.split( string.strip( l ), '\t' )
+      if bits[0] == 'f':
+        self.uf[bits[1]] = dhdl(value=bits[2])
+      else:
+        self.uds[(bits[1], int(bits[2]) )] = dhdl(value=bits[3])
+    oo.close()
+       
+  def saveIds(self):
+    oo = open('cachedDummyHandles.txt', 'w')
+    for k in self.uds:
+      oo.write( 't\t%s\t%s\t%s\n' % (k[0],k[1],self.uds[k].value ) )
+    for k in self.uf:
+      oo.write( 'f\t%s\t%s\n' % (k,self.uf[k].value ) )
+    oo.close()
+    
+  def genDummy(self):
+    dv = {}
+    ddv = {'aggregation_level':'dataset'}
+    ddf = {'aggregation_level':'file'}
+    self.dsd = {}
+    for ds in self.f1.keys():
+      kds=0
+      for t in self.f1[ds]:
+        values = []
+        kds += 1
+        cl = []
+        for fn, cs in t:
+          u = self.uf[cs]
+          cl.append( u.value )
+          if u.value not in self.hh:
+            extr = []
+            ddf['tracking_id'] = u.value
+            ddf['parent'] = self.uds[(ds,kds)].value
+            if cs in self.mp.t:
+              ddf['replaced_by'] = self.uf[ self.mp.t[cs] ].value
+              extr.append( 'replaced_by' )
+              print '>>> %s replaced_by %s' % (cs,ddf['replaced_by'])
+            if cs in self.mp.f:
+              ddf['replaces'] = self.uf[ self.mp.f[cs] ].value
+              extr.append( 'replaces' )
+              print '>>> %s replaces %s' % (cs,ddf['replaces'])
+            thisV = []
+            kk = 0
+            for k in expectedC + extr:
+              kk += 1
+              thisV.append( {'index':kk, 'type':k, 'data':{'format':'string', 'value':ddf.get(k,'dummy')} } )
+            self.hh[u.value] = {"responseCode":1, "handle":u.value, "values":thisV}
+        dv[('children',kds)] = str(cl)
+        dv[('tracking_id',kds)] = self.uds[(ds,kds)].value
+          
+        kk = 0
+        for k in expectedP:
+          kk += 1
+          values.append( {'index':kk, 'type':k, 'data':{'format':'string', 'value':dv.get((k,kds),ddv.get(k,'dummy'))} } )
+
+        if kds < 4:
+          values.append( {'index':kk, 'type':'isReplacedBy', 'data':{'format':'string', 'value':self.uds[(ds,kds+1)].value} } )
+        if kds > 1:
+          values.append( {'index':kk, 'type':'replaces', 'data':{'format':'string', 'value':self.uds[(ds,kds-1)].value} } )
+        self.dsd[(ds,kds)] = {"responseCode":1,"handle":self.uds[(ds,kds)].value, "values":values}
+        self.hh[self.dsd[(ds,kds)]["handle"]] = self.dsd[(ds,kds)]
+      
+      
+    
 
 hsamp2 = {"responseCode":1,"handle":"10876.test/f05ca8f4-f011-11e4-8220-5404a60d96b5","values":[{"index":1,"type":"URL","data":{"format":"string","value":"http://clipc-services.ceda.ac.uk/testdata/v1/novar_fx_dummy_historical_r2i1p1.nc"},"ttl":86400,"timestamp":"2015-06-09T12:46:07Z"},{"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":"0.NA/10876.TEST","index":200,"permissions":"011111110011"}},"ttl":86400,"timestamp":"2015-06-09T12:46:07Z"},{"index":2,"type":"10320/loc","data":{"format":"string","value":"<locations chooseby=\"locatt,weighted\">\n<location ctype=\"text/html\" weight=\"1\" href=\"https://handle8.dkrz.de/landingpagedemo/10876.test/f05ca8f4-f011-11e4-8220-5404a60d96b5\" />\n<location http_role=\"conneg\" weight=\"0\" href=\"http://clipc-services.ceda.ac.uk/testdata/v1/novar_fx_dummy_historical_r2i1p1.nc\" />\n</locations>"},"ttl":86400,"timestamp":"2015-06-10T16:17:03Z"},{"index":3,"type":"creation_date","data":{"format":"string","value":"2015-05-01"},"ttl":86400,"timestamp":"2015-06-10T11:57:02Z"},{"index":4,"type":"tracking_id","data":{"format":"string","value":"f05ca8f4-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-10T11:57:08Z"},{"index":5,"type":"replaced_by","data":{"format":"string","value":"10876.test/f05f7caa-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-10T11:57:14Z"},{"index":6,"type":"checksum","data":{"format":"string","value":"ad99a037946fc7e9eaa1d916c8682a11"},"ttl":86400,"timestamp":"2015-06-09T12:46:07Z"},{"index":7,"type":"parent","data":{"format":"string","value":"10876.test/49634b69-6662-4a52-9175-45f296dc9578"},"ttl":86400,"timestamp":"2015-06-10T11:57:21Z"},{"index":8,"type":"aggregation_level","data":{"format":"string","value":"file"},"ttl":86400,"timestamp":"2015-06-10T11:57:34Z"}]}
 hsample = {"responseCode":1,"handle":"10876.test/f0600a58-f011-11e4-8220-5404a60d96b5","values":[{"index":1,"type":"URL","data":{"format":"string","value":"http://clipc-services.ceda.ac.uk/testdata/v2/novar_fx_dummy_historical_r3i1p1.nc"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":"0.NA/10876.TEST","index":200,"permissions":"011111110011"}},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":2,"type":"10320/loc","data":{"format":"string","value":"<locations chooseby=\"locatt,weighted\">\n<location ctype=\"text/html\" weight=\"1\" href=href=\"https://handle8.dkrz.de/landingpagedemo/10876.test/f0600a58-f011-11e4-8220-5404a60d96b5\" />\n<location http_role=\"conneg\" weight=\"0\" href=\"http://clipc-services.ceda.ac.uk/testdata/v2/novar_fx_dummy_historical_r3i1p1.nc\" />\n</locations>"},"ttl":86400,"timestamp":"2015-06-10T16:21:11Z"},{"index":3,"type":"creation_date","data":{"format":"string","value":"2015-05-01"},"ttl":86400,"timestamp":"2015-06-10T12:01:04Z"},{"index":4,"type":"replaces","data":{"format":"string","value":"10876.test/f05d326a-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":5,"type":"trackingID","data":{"format":"string","value":"f0600a58-f011-11e4-8220-5404a60d96b5"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":6,"type":"checksum","data":{"format":"string","value":"e2e71fe45c106298dc15fbd08f9707e1"},"ttl":86400,"timestamp":"2015-06-09T12:39:53Z"},{"index":7,"type":"parent","data":{"format":"string","value":"10876.test/ca9e9abd-e66e-413e-ab29-6c26fe00b859"},"ttl":86400,"timestamp":"2015-06-10T12:01:10Z"},{"index":8,"type":"aggregation_level","data":{"format":"string","value":"file"},"ttl":86400,"timestamp":"2015-06-10T12:01:21Z"}]}
+
+psampl1 = {"responseCode":1,"handle":"10876.test/49634b69-6662-4a52-9175-45f296dc9578","values":[{"index":1,"type":"URL","data":{"format":"string","value":"http://clipc-services.ceda.ac.uk/testdata/v1"},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":"0.NA/10876.TEST","index":200,"permissions":"011111110011"}},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":2,"type":"10320/loc","data":{"format":"string","value":"<locations chooseby=\"locatt,weighted\">\n<location ctype=\"text/html\" weight=\"1\" href=href=\"https://handle8.dkrz.de/landingpagedemo/10876.test/49634b69-6662-4a52-9175-45f296dc9578\" />\n<location http_role=\"conneg\" weight=\"0\" href=\"http://clipc-services.ceda.ac.uk/testdata/v1/\" />\n</locations>"},"ttl":86400,"timestamp":"2015-06-10T16:24:37Z"},{"index":3,"type":"creation_date","data":{"format":"string","value":"2015-05-01"},"ttl":86400,"timestamp":"2015-06-10T12:04:29Z"},{"index":4,"type":"tracking_id","data":{"format":"string","value":"none"},"ttl":86400,"timestamp":"2015-06-10T12:04:35Z"},{"index":5,"type":"isReplacedBy","data":{"format":"string","value":"10876.test/ca9e9abd-e66e-413e-ab29-6c26fe00b859"},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":6,"type":"checksum","data":{"format":"string","value":"todo"},"ttl":86400,"timestamp":"2015-06-10T08:04:45Z"},{"index":7,"type":"children","data":{"format":"string","value":"[\"10876.test/f05e5f1e-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05d326a-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05ca8f4-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05c1876-f011-11e4-8220-5404a60d96b5\", \"10876.test/f05dc00e-f011-11e4-8220-5404a60d96b5\"]"},"ttl":86400,"timestamp":"2015-06-10T12:04:46Z"},{"index":8,"type":"aggregation_level","data":{"format":"string","value":"dataset"},"ttl":86400,"timestamp":"2015-06-10T12:04:57Z"}]}