source: hdldemo/trunk/hdldemo.py @ 565

"""
Demonstration handle dump for CMIP/ESGF files ..

 -h: print this message;
 -v: print version;
 -t: run a test;
 -f <file name>: examine file, print path to replacement if this file is obsolete, print path to sibling files (or replacements).
 -id <tracking id>: examine handle record of tracking id.
"""
## see https://www.handle.net/proxy_servlet.html for info on restfull API

import string, collections, os
import ncq3
import xml
from xml.dom import minidom

try:
 import urllib
 from urllib import request
except:
 import requests

from testdata import *

class phandle(object):
  
  def __init__(self, hdlDict, k='values'):
    """Obsolete class to parse handle metadat ... replaced by pyhandle class"""
    self.h = hdlDict
    self.d = {}
    try:
      for r in hdlDict[k]:
        self.d[r['type']] = r['data']
    except:
      print ( hdlDict[k] )
      raise

class ghandle(object):
  htmpl = 'http://hdl.handle.net/api/handles/%s'
  dh = dummyHandles()
  def __init__(self,hdl,url=None):
     """Class to retrieve a handle .. optionally to retrieve from test data.
     Still needs some error handling based on 'responseCode'."""

     if hdl[:5] == 'xxxxx':
       self.msg = self.dh.hh[hdl]
     else:
       if url == None:
         thisid = string.replace(hdl,'hdl:999999', '10876.test' )
         url = self.htmpl % thisid
       self.fetch(url)

  def fetch(self,url):

     try:
       fh = request.urlopen( url )
       msg = eval( fh.read() )
     except:
       r = requests.get( url )
       msg = r.json()
     assert type( msg ) == type( {} ), 'Response of wrong type'
     for k in ['responseCode', 'handle']:
       assert k in msg, 'Required key %s not found: %s' % (k, str( msg.keys() ) )

     self.msg = msg

class GHandles(object):
  def __init__(self):
    """Class to manage a local cache of retrieved handles, to avoid repeat http requests"""
    self.hh = {}
  def __call__(self,hdl):
    if hdl not in self.hh:
      self.hh[hdl] = ghandle(hdl)
    return self.hh[hdl]


class pyhandle(object):
  def __init__(self,id):
    self.REC_id = id
    self.REC_got = False

  def get(self):
    if self.REC_got:
      return

    g = ghandles( self.REC_id )
    for r in g.msg['values']:
        if str(r['type']) in ['replaces', 'replacedBy', 'isReplacedBy','parent','replaced_by']:
          self.__dict__[r['type']] = pyhandle( r['data']['value'] )

        elif r['type']  == 'children':
          cl = eval( r['data']['value'] )
          self.__dict__['children'] = [pyhandle(c) for c in cl]

        else:
          self.__dict__[r['type']] = r['data']['value']

    self.obsolete = 'replaced_by' in self.__dict__
    if 'URL' in self.__dict__:
      fn = string.split( self.__dict__['URL'], '/' )[-1]
      self.__dict__['filename'] = fn
    else:
      self.__dict__['filename'] = '__unnamed__'

  def addLatest(self):
    """Retrieve handle records for replacements until a current file is found."""
    if not self.obsolete:
      return
    self.replaced_by.get()
    self.replacements=[self.replaced_by,]
    while self.replacements[-1].obsolete:
        self.replacements.append( self.replacements[-1].replaced_by.get() )
    self.latest = self.replacements[-1]

  def addSiblings(self):
     if self.aggregation_level != 'file':
       print 'No known siblings .....'
       return

     if 'parent' not in self.__dict__:
       print 'No parent'
       return

     self.parent.get()
     self.siblings = []
     for c in self.parent.children:
       if c.REC_id != self.REC_id:
         self.siblings.append( c )

class main(object):
  knownargs0 = ['-h','-v','-t']
  knownargs1 = ['-f','-id']
  def __init__(self, args):
    self.htmpl = 'http://hdl.handle.net/api/handles/%s'
    self.version = '0.01.01'
    self.args = args
    self.parseArgs()
    if self.d.get( '-v', False ):
      print ( self.version )
      return

    if self.d.get( '-h', False ):
      print (self.version)
      print ( __doc__ )
      return

    if self.d.get( '-t', False ):
      self.runTest()
      return

    if '-f' in self.d:
      fn = self.d['-f']
      self.dumpF(fn)

    if '-id' in self.d:
      id = self.d['-id']
      self.dumpF('',id=id)

  def dumpF(self,fn, id=None):
    if id == None:
      assert os.path.isfile( fn ), 'File %s not found' % fn
      f = ncHead( fn )
      thisid = string.replace(f.ga['tracking_id'], 'hdl:999999', '10876.test' )
    else:
      thisid = string.replace(id,'hdl:999999', '10876.test' )

    self.p = pyhandle(  thisid )
    self.p.get()
    if self.p.obsolete:
      print '******* OBSOLETE FILE ******'
      self.p.addLatest()
      for this in self.p.replacements:
        print 'REPLACED BY: ',this.URL, this.REC_id
    else:
      print '****** File is current'

    if 'parent' not in self.p.__dict__:
      print 'No parent'
    else:
      self.p.parent.get()
      print 'Parent: %s' % self.p.parent.REC_id
      self.p.addSiblings()
      print '------- SIBLINGS -------'
      for c in self.p.siblings:
        c.get()
        if c.obsolete:
          c.addLatest()
          print '>>>', c.latest.URL
        else:
          print c.URL

  def dumpF__obsolete(self,fn, id=None):
    if id == None:
      assert os.path.isfile( fn ), 'File %s not found' % fn
      f = ncHead( fn )
      thisid = string.replace(f.ga['tracking_id'], 'hdl:999999', '10876.test' )
    else:
      thisid = string.replace(id,'hdl:999999', '10876.test' )

    url = self.htmpl % thisid
    g = ghandle( thisid )
    self.p = phandle(  g.msg )
    gp = ghandle(  self.p.d['parent']['value'] )
    self.pp = phandle(  gp.msg )
    isReplaced = 'replaced_by' in self.p.d
    if isReplaced:
      print '******* OBSOLETE FILE ******'
      pr = self.p
      while isReplaced:
        h2 = pr.d['replaced_by']['value']
        pr = phandle(  ghandle( pr.d['replaced_by']['value'] ).msg )
        isReplaced = 'replaced_by' in pr.d
        print 'REPLACED BY: ',pr.d['URL']['value'],h2
    else:
      print 'File is current'

    cl = eval( self.pp.d['children']['value'] )
    thisfound=False
    print '------- SIBLINGS -------'
    for c in cl:
      if c == thisid:
        thisfound = True
      else:
        gc = ghandle(  c )
        pc = phandle( gc.msg )
        isReplaced = 'replaced_by' in pc.d
        rsq = [pc,]
        if isReplaced:
          while isReplaced:
            pc = phandle(  ghandle( pc.d['replaced_by']['value'] ).msg )
            isReplaced = 'replaced_by' in pc.d
            print '>>>', pc.d['URL']['value']
        elif 'URL' in pc.d:
          print pc.d['URL']['value']
        else:
          print 'Sibling url not found',pc.d.keys()

  def runTest(self):
    f = ncHead( fn1 )
    url = self.htmpl % f.ga['tracking_id']
    hdl = string.replace( f.ga['tracking_id'], 'hdl:999999', '10876.test' )
    g = ghandle( hdl )
    self.p = phandle(  g.msg )
    expected= ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'parent']
    ## 'replaced_by' if obsolete
    for k in expected:
      assert k in self.p.d, 'Expected handle content key %s not found:: %s' % (k,str(self.p.d.keys()))
    assert 'tracking_id' in self.p.d or 'trackingID' in self.p.d, 'No tracking id found: %s' % str(self.p.d.keys())
    for k in expected:
      print ('%s: %s' % (k,self.p.d[k]))

    print ('PARSING PARENT ..... ' )
    print ( self.htmpl % self.p.d['parent']['value'] )
    g = ghandle( self.p.d['parent']['value'] )
    self.pp = phandle(  g.msg )
    for k in self.pp.d.keys():
      print ('%s: %s' % (k,self.pp.d[k]))
    #'isReplacedBy' if obsolete
    expected= ['creation_date', 'aggregation_level', 'HS_ADMIN', '10320/loc', 'checksum', 'URL', 'children', 'tracking_id']

  def parseArgs(self):
    self.d = {}
    kn = self.knownargs0 + self.knownargs1
    xx = []
    al = self.args[1:]
    while len(al) > 0:
      a = al.pop(0)
      if a not in kn:
        xx.append(a)
      elif a in self.knownargs1:
         self.d[a] = al.pop(0)
      else:
         self.d[a] = True
    if len(xx) > 0:
      print ('ARGUMENTS NOT RECOGNISED: %s' % str(xx) )
        
class ncHead(object):
  def __init__(self, fn):
    """Read global attributes of a NetCDF file"""
    nc0 = ncq3.open( fn )
    nc0.getDigest()
    self.ga = {}
    for a in nc0.alla:
      self.ga[a.name] = a.value
      
ghandles = GHandles()
if __name__ == "__main__":
  import sys
  m = main( sys.argv )