Changeset 180 for CCCC/trunk


Ignore:
Timestamp:
30/06/14 12:08:11 (6 years ago)
Author:
mjuckes
Message:

merged from branches-mnj (which was branched from branches-sp)

Location:
CCCC/trunk
Files:
19 deleted
35 edited
6 copied

Legend:

Unmodified
Added
Removed
  • CCCC/trunk

  • CCCC/trunk/README.txt

    r151 r180  
    77Required arguments: 
    88 
    9 python c4.py -p <project> -D <directory>  ## check all files in directory tree, for project in SPECS, CORDEX, CCMI, CMIP5. 
    10 python c4.py -p <project> -d <directory>  ## check all files in directory 
    11 python c4.py -p <project> -f <file>       ## check a single file. 
     9python ceda_cc/c4.py -p <project> -D <directory>  ## check all files in directory tree, for project in SPECS, CORDEX, CCMI, CMIP5. 
     10python ceda_cc/c4.py -p <project> -d <directory>  ## check all files in directory 
     11python ceda_cc/c4.py -p <project> -f <file>       ## check a single file. 
     12python ceda_cc/c4.py --copy-config <dest-dir>     ## copy the default configuration directory to <dest-dir> to enable customisation. 
    1213 
    1314Optional arguments: 
     
    2324  --aMap                # Read in some attribute mappings and run tests with virtual substitutions, see also map2nco.py 
    2425 
     26Environment variables: 
     27 
     28  CC_CONFIG_DIR  ## Set to the location of a custom configuration directory.  If unset the default configuration will be used. 
     29 
    2530After running: 
    2631 
     
    3136This will produce a listing of errors, the number of times they occur and up to two of the files which contain the error. It is hoped that inspection of one or 2 files will provide enough information to trace the problems which lead to the error reports. 
    3237 
     38 
     39Installing as a package: 
     40------------------------ 
     41 
     42You can also install the code into your Python environment and then use the "ceda-cc" command to invoke c4.py with the same arguments ans described above. 
     43 
     44 1. If you have "pip" installed simply execute: 
     45    $ pip install ceda-cc 
     46    or after downloading the tarball 
     47    $ pip install ceda-cc-$VERSION.tar.gz 
     48 
     49 2. If you have the setuptools package you can execute the following from the distribution directory: 
     50    $ python setup.py install 
     51 
     52If you install ceda-cc in this way you can use the --copy-config command to export the default configuration into a directory where you can edit the configuration. 
     53 
     54 
    3355Called from python: 
    3456------------------ 
    3557The code can also be called from a python script: 
    3658 
    37 import c4 
     59from ceda_cc import c4 
    3860m = c4.main( args=argList )     # argList is a python list of command line arguments 
    3961if not m.ok: 
     
    83105 
    84106To update the CMOR tables use:  
    85 "git clone git://uv-cdat.llnl.gov/gitweb/cordex-cmor-tables.git" 
     107"git clone https://github.com/PCMDI/cordex-cmor-tables" 
    86108 
    87109VIRTUAL MODE 
  • CCCC/trunk/ceda_cc/amap2nco.py

    r160 r180  
    11 
    22import os, string, random, time 
     3from xceptions import * 
    34 
    45xx='abcdefghijklmnopqrstuvwxyz1234567890$%-=+*' 
    56 
    6 class map2nco: 
     7class map2nco(object): 
    78 
    89 
     
    2223          self.directives.append( ('ga',l[2:] ) ) 
    2324        else: 
    24           raise 'unrecognised directive:\n %s' % l 
     25          raise baseException( 'unrecognised directive:\n %s' % l ) 
    2526      elif l[0] != '#': 
    2627        if string.strip(l) != '': 
    27            raise 'unrecognised line:\n %s' % l 
     28           raise baseException( 'unrecognised line:\n %s' % l ) 
    2829    self.ipth = ipth 
    2930    self.opth = opth 
  • CCCC/trunk/ceda_cc/c4.py

    r167 r180  
     1 
     2import sys 
     3 
     4## callout to summary.py: if this option is selected, imports of libraries are not needed. 
     5if __name__ == '__main__' and sys.argv[1] == '--sum': 
     6      import summary 
     7      summary.main() 
     8      raise SystemExit(0) 
    19 
    210# Standard library imports 
    311import os, string, time, logging, sys, glob, pkgutil 
    412import shutil 
    5  
    613## pkgutil is used in file_utils 
    714# Third party imports 
     
    2027reload( utils ) 
    2128 
     29from xceptions import baseException 
     30 
    2231 
    2332#driving_model_ensemble_member = <CMIP5Ensemble_member> 
    2433#rcm_version_id = <RCMVersionID>                      
    2534 
    26 class dummy: 
     35class dummy(object): 
    2736   pass 
    2837 
     
    3342               } 
    3443 
    35 class recorder: 
     44class recorder(object): 
    3645 
    3746  def __init__(self,project,fileName,type='map',dummy=False): 
     
    101110    self.records[fn] = record 
    102111 
    103 class checker: 
     112class checker(object): 
    104113  def __init__(self, pcfg, cls,reader,abortMessageCount=-1): 
    105114    self.info = dummy() 
     
    183192    self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount 
    184193 
    185 class c4_init: 
     194class c4_init(object): 
    186195 
    187196  def __init__(self,args=None): 
     
    211220       print 
    212221       raise SystemExit(0) 
     222 
     223    self.summarymode = args[0] == '--sum' 
     224    if self.summarymode: 
     225      return 
    213226 
    214227    while len(args) > 0: 
     
    378391 
    379392 
    380 class main: 
     393class main(object): 
    381394 
    382395  def __init__(self,args=None,abortMessageCount=-1,printInfo=False,monitorFileHandles = False): 
     
    384397    ecount = 0 
    385398    c4i = c4_init(args=args) 
     399       
    386400    isDummy  = c4i.project[:2] == '__' 
    387     if (ncLib == dummy) and (not isDummy): 
    388        print ncLib, c4i.project 
    389        print 'Cannot proceed with non-dummy project without cdms' 
    390        raise 
     401    if (ncLib == None) and (not isDummy): 
     402       raise baseException( 'Cannot proceed with non-dummy [%s] project without a netcdf API' % (c4i.project) ) 
    391403    pcfg = config.projectConfig( c4i.project ) 
    392404    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog) 
     
    519531 
    520532if __name__ == '__main__': 
     533  if sys.argv[1] == '--sum': 
     534      import summary 
     535      summary.main() 
     536      raise SystemExit(0) 
    521537  main_entry() 
    522538 
  • CCCC/trunk/ceda_cc/comp_mip.py

    r162 r180  
    4444   return ee 
    4545 
    46 class comp: 
     46class comp(object): 
    4747 
    4848  def __init__(self, snl, snla=None, ec1=None,tag=None): 
     
    156156         
    157157base=CC_CONFIG_DIR 
    158 snl,snla = gen_sn_list( base + cfsntab ) 
     158print base 
     159snl,snla = gen_sn_list( os.path.join(base, cfsntab) ) 
    159160print 'Len snl = %s' % len(snl) 
    160161 
     
    163164if newMip == 'CORDEX': 
    164165 if dkrz_cordex_version == 1: 
    165   ll = open( base + cordex_dkrz ).readlines() 
     166  ll = open( os.path.join(base, cordex_dkrz) ).readlines() 
    166167  for l in ll[9:74]: 
    167168    bits = string.split( l, ',' ) 
     
    174175 elif dkrz_cordex_version == 2: 
    175176  for tab in ['3hr','6hr','day','mon','sem','fx']: 
    176      ll = open( base + cordex_dkrz_pat % tab ).readlines() 
     177     ll = open( os.path.join(base, cordex_dkrz_pat % tab) ).readlines() 
    177178     for l in ll[3:]: 
    178179        bits = string.split( l, ',' ) 
     
    191192  eeee = {} 
    192193  eca = {} 
    193   ll = open( base + cordex_dkrz_pat % 'all' ).readlines() 
     194  ll = open( os.path.join(base, cordex_dkrz_pat % 'all') ).readlines() 
    194195  for l in ll[2:]: 
    195196        bits = string.split( string.strip(l), ',' ) 
     
    208209  for tab in ['3hr','6hr','day','mon','sem','fx']: 
    209210     ee  = {} 
    210      ll = open( base + cordex_dkrz_pat % tab ).readlines() 
     211     ll = open( os.path.join(base, cordex_dkrz_pat % tab) ).readlines() 
    211212     for l in ll[2:]: 
    212213        bits = string.split( l, ',' ) 
     
    224225 
    225226if newMip == 'SPECS': 
    226   newMipDir = '/specs_vocabs/mip/' 
     227  newMipDir = 'specs_vocabs/mip/' 
    227228  mpat = 'SPECS_%s' 
    228229  ml = ['SPECS_Amon', 'SPECS_fx', 'SPECS_Lmon', 'SPECS_Omon', 'SPECS_6hrLev', 'SPECS_day', 'SPECS_OImon'] 
     
    230231  ml = ['CCMI1_Amon_v2_complete'] 
    231232  ml = ['CCMI1_annual_comp-v3.txt', 'CCMI1_daily_comp-v3.txt', 'CCMI1_fixed_comp-v2.txt', 'CCMI1_hourly_comp-v3.txt', 'CCMI1_monthly_comp-v3.txt'] 
    232   newMipDir = '/ccmi_vocabs/mip/' 
     233  newMipDir = 'ccmi_vocabs/mip/' 
    233234  mpat = 'CCMI1_%s_v1_complete' 
    234235elif newMip == 'CORDEX': 
    235   newMipDir = '/cordex_vocabs/mip/' 
     236  newMipDir = 'cordex_vocabs/mip/' 
    236237  mpat = 'CORDEX_%s' 
    237238 
     
    241242    l2 = {} 
    242243    for m in ml: 
    243       l1 = ms.scan_table( open( base + newMipDir + m ).readlines(), None, asDict=True, appendTo=l1, lax=True, tag=m, project=newMip) 
     244      print 'base: ',base 
     245      l1 = ms.scan_table( open( os.path.join(base, newMipDir + m) ).readlines(), None, asDict=True, appendTo=l1, lax=True, tag=m, project=newMip) 
    244246    ms.nn_tab = len(ml) 
    245247    for m in ml2: 
    246       l2 = ms.scan_table( open( base + '/cmip5_vocabs/mip/' + m ).readlines(), None, asDict=True, appendTo=l2, lax=True, tag=m, warn=False) 
     248      l2 = ms.scan_table( open( os.path.join(base, 'cmip5_vocabs/mip/' + m) ).readlines(), None, asDict=True, appendTo=l2, lax=True, tag=m, warn=False) 
    247249    ms.nn_tab2 = len(ml2) 
    248250    ccm = False 
     
    251253    l2 = {} 
    252254    for m in ml2: 
    253       l2 = ms.scan_table( open( base + '/cmip5_vocabs/mip/' + m ).readlines(), None, asDict=True, appendTo=l2, lax=True, tag=m, warn=False) 
     255      l2 = ms.scan_table( open( os.path.join(base, 'cmip5_vocabs/mip/' + m) ).readlines(), None, asDict=True, appendTo=l2, lax=True, tag=m, warn=False) 
    254256    k = { '3hr':'3h', '6hr':'6h' }.get( t,t ) 
    255     l1 = ms.scan_table( open( base + newMipDir + mpat % k ).readlines(), None, asDict=True, project=newMip) 
     257    l1 = ms.scan_table( open( os.path.join(base, newMipDir + mpat % k) ).readlines(), None, asDict=True, project=newMip) 
    256258    ccm = True 
    257259    tag = t 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_3hr

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
     
    259259cell_methods:      time: point 
    260260cell_measures:     area: areacella 
    261 long_name:         Air Temperature 
     261long_name:         Near-Surface Air Temperature 
    262262comment:           This is sampled synoptically. 
    263263!---------------------------------- 
     
    572572cell_measures:     area: areacella 
    573573long_name:         Snowfall Flux 
    574 comment:           at surface.  Includes all forms of precipitating solid phase of water.  This is the 3-hour mean snowfall flux. 
     574comment:           at surface.  Includes precipitation of all forms water in the solid phase.  This is the 3-hour mean snowfall flux. 
    575575!---------------------------------- 
    576576! Additional variable information: 
     
    611611! Variable attributes: 
    612612!---------------------------------- 
    613 standard_name:     downwelling_longwave_flux_in_air_assuming_clear_sky 
     613standard_name:     surface_downwelling_longwave_flux_in_air_assuming_clear_sky 
    614614units:             W m-2 
    615615cell_methods:      time: mean 
     
    684684cell_methods:      time: point 
    685685cell_measures:     area: areacella 
    686 long_name:         Surface Pressure 
     686long_name:         Surface Air Pressure 
    687687comment:           sampled synoptically to diagnose atmospheric tides, this is better than mean sea level pressure. 
    688688!---------------------------------- 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_6hrLev

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_6hrPlev

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_Amon

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_LImon

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_Lmon

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_OImon

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
     
    392392cell_measures:     area: areacello 
    393393long_name:         Surface Snowfall Rate into the Sea Ice Portion of the Grid Cell 
    394 comment:           where sea ice over sea: this is computed as the the water mass per unit time falling as snow onto the sea ice portion of a grid cell divided by the area of the ocean portion of the grid cell (including both ice-free and sea-ice covered fractions).  Reported as 0. in regions free of sea ice. 
     394comment:           where sea ice over sea: this is computed as the total mass per unit time of solid-phase precipitation falling onto the sea ice portion of a grid cell divided by the area of the ocean portion of the grid cell (including both ice-free and sea-ice covered fractions).  Reported as 0. in regions free of sea ice. (Snowfall flux includes all types of solid-phase precipitation.) 
    395395!---------------------------------- 
    396396! Additional variable information: 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_Oclim

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_Omon

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
     
    38403840cell_measures:     area: areacello 
    38413841long_name:         Snowfall Flux where Ice Free Ocean over Sea 
    3842 comment:           computed as the total mass of ice directly falling as snow into the ice-free portion of the ocean divided by the area of the ocean portion of the grid cell. 
     3842comment:           computed as the total mass per unit time of solid-phase precipitation falling into the ice-free portion of the ocean divided by the area of the ocean portion of the grid cell.   (Snowfall flux includes all types of solid-phase precipitation.) 
    38433843!---------------------------------- 
    38443844! Additional variable information: 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_Oyr

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_aero

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_cf3hr

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
     
    23202320cell_measures:     area: areacella 
    23212321long_name:         Convective Snowfall Flux 
     2322comment:           convective precipitation of all forms of water in the solid phase. 
    23222323!---------------------------------- 
    23232324! Additional variable information: 
     
    23412342cell_measures:     area: areacella 
    23422343long_name:         Stratiform Snowfall Flux 
     2344comment:           large-scale precipitation of all forms of water in the solid phase. 
    23432345!---------------------------------- 
    23442346! Additional variable information: 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_cfDay

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_cfMon

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_cfOff

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_cfSites

    r163 r180  
    1212cf_version:   1.4         ! version of CF that output conforms to 
    1313project_id:   CMIP5  ! project id 
    14 table_date:   22 March 2012 ! date this table was constructed 
     14table_date:   17 July 2013 ! date this table was constructed 
    1515 
    1616missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_day

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
     
    649649cell_methods:      time: mean 
    650650cell_measures:     area: areacella 
    651 long_name:         Solid Precipitation 
     651long_name:         Snowfall Flux 
    652652comment:           at surface; includes precipitation of all forms of water in the solid phase 
    653653!---------------------------------- 
  • CCCC/trunk/ceda_cc/config/cmip5_vocabs/mip/CMIP5_fx

    r163 r180  
    77cf_version:   1.4         ! version of CF that output conforms to 
    88project_id:   CMIP5  ! project id 
    9 table_date:   22 March 2012 ! date this table was constructed 
     9table_date:   17 July 2013 ! date this table was constructed 
    1010 
    1111missing_value: 1.e20      ! value used to indicate a missing value 
  • CCCC/trunk/ceda_cc/config/cordex_vocabs/mip/CORDEX_6h

    r163 r180  
    12181218 
    12191219 
     1220!============ 
     1221variable_entry:    snc 
     1222!============ 
     1223deflate: 1 
     1224deflate_level: 1 
     1225shuffle: 1 
     1226modeling_realm:    landIce land 
     1227!---------------------------------- 
     1228! Variable attributes: 
     1229!---------------------------------- 
     1230standard_name:     surface_snow_area_fraction 
     1231units:             % 
     1232cell_methods:      time: point 
     1233 
     1234long_name:         Snow Area Fraction 
     1235comment:           Fraction of each grid cell that is occupied by snow that rests on land portion of cell. 
     1236!---------------------------------- 
     1237! Additional variable information: 
     1238!---------------------------------- 
     1239dimensions:        longitude latitude time1 
     1240out_name:          snc 
     1241type:              real 
     1242!---------------------------------- 
     1243! 
     1244 
     1245!============ 
     1246variable_entry:    snd 
     1247!============ 
     1248deflate: 1 
     1249deflate_level: 1 
     1250shuffle: 1 
     1251modeling_realm:    landIce land 
     1252!---------------------------------- 
     1253! Variable attributes: 
     1254!---------------------------------- 
     1255standard_name:     surface_snow_thickness 
     1256units:             m 
     1257cell_methods:      time: point area: mean where land 
     1258 
     1259long_name:         Snow Depth 
     1260comment:           where land over land.  Compute the mean thickness of snow in the land portion of the grid cell (averaging over the entire land portion, including the snow-free fraction.  Report as 0.0 where the land fraction is 0. 
     1261!---------------------------------- 
     1262! Additional variable information: 
     1263!---------------------------------- 
     1264dimensions:        longitude latitude time1 
     1265out_name:          snd 
     1266type:              real 
     1267!---------------------------------- 
     1268! 
     1269 
     1270 
  • CCCC/trunk/ceda_cc/config/cordex_vocabs/mip/CORDEX_mon

    r163 r180  
    485485standard_name:     duration_of_sunshine 
    486486units:             s 
    487 cell_methods:      time: sum 
     487cell_methods:      time: sum within days time: mean over days 
    488488long_name:         Duration of Sunshine 
    489489comment:              
     
    11311131! Additional variable information: 
    11321132!---------------------------------- 
    1133 dimensions:        longitude latitude time 
     1133dimensions:        longitude latitude time plev500 
    11341134out_name:          zg500 
    11351135type:              real 
  • CCCC/trunk/ceda_cc/config/cordex_vocabs/mip/CORDEX_sem

    r163 r180  
    485485standard_name:     duration_of_sunshine 
    486486units:             s 
    487 cell_methods:      time: sum 
     487cell_methods:      time: sum within days time: mean over days 
    488488long_name:         Duration of Sunshine 
    489489comment:              
     
    11311131! Additional variable information: 
    11321132!---------------------------------- 
    1133 dimensions:        longitude latitude time 
     1133dimensions:        longitude latitude time plev500 
    11341134out_name:          zg500 
    11351135type:              real 
  • CCCC/trunk/ceda_cc/config_c4.py

    r167 r180  
    108108    interpolatedGrids[bits[1]] = ee 
    109109 
    110 class readVocab: 
     110class readVocab(object): 
    111111 
    112112  def __init__(self,dir): 
     
    185185  return vocabs 
    186186 
    187 class projectConfig: 
     187class projectConfig(object): 
    188188 
    189189  def __init__(self, project): 
  • CCCC/trunk/ceda_cc/extractMipInfo.py

    r162 r180  
    11 
    22import collections, glob, string 
    3 from fcc_utils2 import mipTableScan 
     3from fcc_utils2 import mipTableScan, snlist 
    44from config_c4 import CC_CONFIG_DIR 
    55 
    66ms = mipTableScan() 
    7  
     7snc = snlist() 
     8 
     9snl, snla = snc.gen_sn_list( ) 
    810NT_mip = collections.namedtuple( 'mip',['label','dir','pattern'] ) 
     11vlist = [ 
     12('uas', 
     13"eastward_wind", 
     14'Eastward Near-Surface Wind Speed', 
     15'Eastward Near-Surface Wind', 
     16'WRONG LONG NAME (*speed* included)'), 
     17('vas', 
     18"northward_wind", 
     19'Northward Near-Surface Wind Speed', 
     20'Northward Near-Surface Wind', 
     21'WRONG LONG NAME (*speed* included)'), 
     22("snc", 
     23"surface_snow_area_fraction", 
     24"Snow Area Fraction", 
     25"Surface Snow Area Fraction", 
     26"WRONG LONG NAME (*surface* omitted)"), 
     27("prsn", 
     28"snowfall_flux",  
     29"Solid Precipitation", 
     30"Snowfall Flux", 
     31"WRONG LONG NAME"), 
     32("tntscpbl", 
     33"tendency_of_air_temperature_due_to_stratiform_cloud_and_precipitation_and_boundary_layer_mixing", 
     34"Tendency of Air Temperature due to Stratiform Cloud Condensation and Evaporation", 
     35"Tendency of Air Temperature Due to Stratiform Cloud and Precipitation and Boundary Layer Mixing", 
     36'WRONG LONG NAME'), 
     37("tas", 
     38"air_temperature", 
     39"Air Temperature", 
     40"Near-Surface Air Temperature", 
     41'WRONG LONG NAME'), 
     42("cfadDbze94", 
     43"histogram_of_equivalent_reflectivity_factor_over_height_above_reference_ellipsoid", 
     44"CloudSat Radar Reflectivity CFAD", 
     45"CloudSat Radar Reflectivity", 
     46"INCONSISTENT LONG NAME"), 
     47("cfadLidarsr532", 
     48"histogram_of_backscattering_ratio_over_height_above_reference_ellipsoid", 
     49"CALIPSO Scattering Ratio CFAD", 
     50"CALIPSO Scattering Ratio", 
     51"INCONSISTENT LONG NAME"), 
     52("cl", 
     53"cloud_area_fraction_in_atmosphere_layer", 
     54"Cloud Area Fraction", 
     55"Cloud Area Fraction in Atmosphere Layer", 
     56"INCONSISTENT LONG NAME"), 
     57("clcalipso", 
     58"cloud_area_fraction_in_atmosphere_layer", 
     59"CALIPSO Cloud Fraction", 
     60"CALIPSO Cloud Area Fraction", 
     61'WRONG LONG NAME'), 
     62("cltisccp", 
     63"cloud_area_fraction", 
     64"ISCCP Total Total Cloud Fraction", 
     65"ISCCP Total Cloud Fraction", 
     66'WRONG LONG NAME'), 
     67("reffclwc", 
     68"effective_radius_of_convective_cloud_liquid_water_particle", 
     69"Convective Cloud Droplet Effective Radius", 
     70"Hydrometeor Effective Radius of Convective Cloud Liquid Water", 
     71'INCONSISTENT LONG NAMES'), 
     72("reffclws", 
     73"effective_radius_of_stratiform_cloud_liquid_water_particle", 
     74'Stratiform Cloud Droplet Effective Radius', 
     75'Hydrometeor Effective Radius of Stratiform Cloud Liquid Water', 
     76'INCONSISTENT LONG NAMES' ) ] 
     77 
     78class snsub: 
     79 
     80  def __init__(self): 
     81    pass 
     82 
     83  def isFalseSn(self,var,sn): 
     84    if sn == 'mole_concentration_of_molecular_oxygen_in_sea_water': 
     85      return (True,'mole_concentration_of_dissolved_molecular_oxygen_in_sea_water', 'INVALID STANDARD NAME') 
     86    elif var == 'rldscs' and sn == 'downwelling_longwave_flux_in_air_assuming_clear_sky': 
     87      return (True,'surface_downwelling_longwave_flux_in_air_assuming_clear_sky','WRONG STANDARD NAME (should be surface)' ) 
     88    elif var == 'clisccp' and sn == 'cloud_area_fraction_in_atmosphere_layer': 
     89      return (True, 'isccp_cloud_area_fraction', 'WRONG STANDARD NAME (should be isccp)' ) 
     90    return (False,'no match','') 
     91 
     92  def isFalseLn(self,var,ln): 
     93    for tt in vlist: 
     94       if var == tt[0] and ln == tt[2]: 
     95         return (True, tt[3], tt[4] ) 
     96    return (False,'no match','') 
     97 
     98snsubber = snsub() 
    999 
    10100mips = ( NT_mip( 'cmip5','cmip5_vocabs/mip/', 'CMIP5_*' ), ) 
    11101 
    12 cmip5_ignore = ['depth','depth_c','eta','nsigma','vertices_latitude','vertices_longitude','ztop','ptop','p0','z1','z2','href','k_c','a','a_bnds','ap','ap_bnds','b','b_bnds','sigma','sigma_bnds','zlev','zlev_bnds','zfull','zhalf'] 
     102cmip5_ignore = ['pfull','phalf','depth','depth_c','eta','nsigma','vertices_latitude','vertices_longitude','ztop','ptop','p0','z1','z2','href','k_c','a','a_bnds','ap','ap_bnds','b','b_bnds','sigma','sigma_bnds','zlev','zlev_bnds','zfull','zhalf'] 
    13103 
    14104vl0 = [] 
    15105for mip in mips: 
     106 ## dl = glob.glob( '%s%s' % (mip.dir,mip.pattern) ) 
    16107  dl = glob.glob( '%s/%s%s' % (CC_CONFIG_DIR, mip.dir,mip.pattern) ) 
    17108  dl.sort() 
     
    20111  for d in dl: 
    21112    tab = string.split( d, '/')[-1] 
     113    isoceanic = tab[:7] == "CMIP5_O" 
    22114    l2 = ms.scan_table( open( d ).readlines(), None, asDict=True, lax=True, tag="x", warn=True) 
    23115    l2k = [] 
     
    27119    l2k.sort() 
    28120    vl0 += l2k 
    29     tl.append( [tab,l2, l2k] ) 
     121    tl.append( [tab,l2, l2k,isoceanic] ) 
    30122    td[tab] = l2 
    31123 
     
    49141  l = vdict[v] 
    50142  l.sort() 
    51   print v, l, td[l[0]][v][1].get('standard_name','__NO_STANDARD_NAME__') 
    52  
     143##  print v, l, td[l[0]][v][1].get('standard_name','__NO_STANDARD_NAME__') 
     144 
     145vd2 = {} 
     146for v in vars: 
     147  l = vdict[v] 
     148  l.sort() 
     149  if len(l) > 1: 
     150    for att in ['standard_name','units','long_name']: 
     151    ##for att in ['standard_name','units']: 
     152      atl = map( lambda x: td[x][v][1].get(att,'__ABSENT__'), l ) 
     153      atl.sort() 
     154      av = [atl[0],] 
     155      for a in atl[1:]: 
     156        if a != av[-1]: 
     157          av.append(a) 
     158      if att == 'standard_name': 
     159        for a in av: 
     160          if a not in snl and a not in snla: 
     161            print "INVALID STANDARD NAME: ",a 
     162      if len(av) > 1: 
     163        ee = {} 
     164 
     165        for a in [True,False]: 
     166          ee[a] = [] 
     167 
     168        isol = [] 
     169        for x in l: 
     170          a = td[x][v][1].get(att,'__ABSENT__') 
     171          if att == 'standard_name' or ( att == 'long_name' and vd2[v][0] == 2): 
     172            iso = x[:7] == 'CMIP5_O' 
     173            tt = snsubber.isFalseSn( v, a ) 
     174          elif att == 'long_name': 
     175            tt = snsubber.isFalseLn( v, a ) 
     176            dims = td[x][v][0] 
     177            iso = 'depth0m' in dims 
     178          else: 
     179            iso = False 
     180            tt = (False,) 
     181      ##    iso = False 
     182          isol.append((iso,x)) 
     183          if tt[0]: 
     184            print 'Substituting ',v,a,tt 
     185            ee[iso].append( tt[1] ) 
     186          else: 
     187            ee[iso].append( a ) 
     188 
     189        for a in [True,False]: 
     190          ok = True 
     191          if len(ee[a]) > 1 : 
     192            for x in ee[a][1:]: 
     193              if x != ee[a][0]: 
     194                ok = False 
     195 
     196          if not ok: 
     197             print 'Multiple values : ',att,v 
     198             for t in isol: 
     199               if t[0] == a: 
     200                 tab = t[1] 
     201                 print tab,td[tab][v][1].get('standard_name','__ABSENT__'),td[tab][v][1].get('long_name','__ABSENT__') 
     202                 
     203        if att == "standard_name": 
     204          vd2[v] = (2,[ee[True],ee[False]] ) 
     205      else: 
     206        if att == "standard_name": 
     207          tt = snsubber.isFalseSn( v, av[0] ) 
     208          if tt[0]: 
     209            print 'Substituting ',v,av[0],tt 
     210            vd2[v] = (1, tt[1]) 
     211          else: 
     212            vd2[v] = (1, av) 
     213  else: 
     214        tab = vdict[v][0] 
     215        a = td[tab][v][1].get('standard_name','__ABSENT__') 
     216        tt = snsubber.isFalseSn( v, a ) 
     217        if tt[0]: 
     218          print 'Substituting ',v,a,tt 
     219          vd2[v] = (1, tt[1]) 
     220        else: 
     221          vd2[v] = (1, a) 
     222        ##print 'MULTIPLE VALUES: ',v,att,av 
     223    
     224 
     225 
     226 
  • CCCC/trunk/ceda_cc/fcc_utils.py

    r160 r180  
    55## 
    66 
    7 class mipTableScan: 
     7from xceptions import * 
     8 
     9class mipTableScan(object): 
    810 
    911  def __init__(self, vats = ['standard_name','long_name','units','cell_methods'] ): 
     
    7577             log.warn(  'Mistake?? in scan_table %s' % str(mm) ) 
    7678          ds = mm 
    77           raise 'Mistake?? in scan_table %s' % str(mm) 
     79          raise baseException( 'Mistake?? in scan_table %s' % str(mm) ) 
    7880        ee.append( (var,ds,aa,tag) ) 
    7981 
     
    106108## this class carries a logging method, and is used to carry information about datasets being parsed. 
    107109## 
    108 class qcHandler: 
     110class qcHandler(object): 
    109111 
    110112  def __init__( self, qcc, log, baseDir, logPasses=True ): 
     
    179181    self.msgk[key] += 1 
    180182 
    181 class dirParser: 
     183class dirParser(object): 
    182184 
    183185  def __init__(self, qcc, linksOnly=True): 
     
    331333         else: 
    332334           print 'segment test id %s not recognised' % qfns.segments[k][0] 
    333            raise 'segment test id %s not recognised' % qfns.segments[k][0] 
     335           raise baseException( 'segment test id %s not recognised' % qfns.segments[k][0] ) 
    334336################################## 
    335337       versionned = False 
     
    343345            except: 
    344346              print 'Failed to get dsID:',fbits,dbits 
    345               raise 
     347              raise baseException( 'Failed to get dsID: %s,%s' % (fbits,dbits) ) 
    346348          else: 
    347349            assert False, 'datasetIdMethod %s not supported yet' % self.qcc.datasets[k].datasetIdMethod 
     
    353355          handler.datasets[k][dsId].append( (dir,f, handler.nofail, ns) ) 
    354356 
    355 class dataSetParser: 
     357class dataSetParser(object): 
    356358 
    357359  def __init__(self,qcc, log, handler): 
     
    369371      except: 
    370372        print self.qcc.fileNameSegments.segments 
    371         raise 
     373        raise baseException(  str( self.qcc.fileNameSegments.segments ) ) 
    372374      dsok = True 
    373375      for dir,f, fok, ns in files: 
     
    521523        
    522524 
    523 class dataset: 
     525class dataset(object): 
    524526   def __init__(self,name): 
    525527     self.name = name 
    526528 
    527 class qcConfigParse: 
     529class qcConfigParse(object): 
    528530 
    529531  def __init__( self, file, log=None ): 
     
    594596revsc = re.compile( 'validSectionCount,(.+)' ) 
    595597 
    596 class section_parser_l1: 
     598class section_parser_l1(object): 
    597599 
    598600  def __init__(self,parent): 
     
    712714                       print 'Error trying to compile: ', v % self.gc 
    713715                       print 'Pattern: ',v 
    714                        raise 
     716                       raise baseException(  'Error trying to compile: %s [%s]' % ( v % self.gc, v) ) 
    715717                     ## print 'compiled ' +  v % self.gc, tt 
    716718                 cr.append( (cc,cr0) ) 
     
    907909          self.parent.datasets[bb[k]].child = self.parent.datasets[bb[k+1]] 
    908910           
    909 class dsid1: 
     911class dsid1(object): 
    910912 
    911913  def __init__(self,slice,sep): 
     
    916918    return string.join( x[self.slice], self.sep )  
    917919 
    918 class cmip5_dsid: 
     920class cmip5_dsid(object): 
    919921 
    920922  def __init__(self,slice,sep): 
     
    926928 
    927929 
    928 class get_trange: 
     930class get_trange(object): 
    929931  
    930932  def __init__(self,pat,kseg): 
     
    974976    return self._get( l[self.kseg] ) 
    975977          
    976 class fileNameSegments: 
     978class fileNameSegments(object): 
    977979  def __init__(self, parent, sep, nn ): 
    978980    self.sep = sep 
     
    994996      except: 
    995997        print 'Failed to compile (in re): %s' % bits[3] 
    996         raise 
     998        raise baseException(  'Failed to compile (in re): %s' % bits[3] ) 
    997999      self.__segments[k] = (bits[1],bits[2], regex) 
    9981000    else: 
     
    10091011    self.segments = tuple( sl ) 
    10101012 
    1011 class Constraint__CordexInterpolatedGrid: 
     1013class Constraint__CordexInterpolatedGrid(object): 
    10121014 
    10131015  def __init__(self): 
     
    10291031    return ('PASS',self.msg) 
    10301032 
    1031 class Constraint__IdentityChecker: 
     1033class Constraint__IdentityChecker(object): 
    10321034 
    10331035  def __init__(self, ref1, ref2 ): 
     
    11001102           return ('VALUE', bits[1]) 
    11011103 
    1102 class Constraint__OnlyOnce: 
     1104class Constraint__OnlyOnce(object): 
    11031105 
    11041106  def __init__(self, ref1): 
     
    11261128#### Mip table variable attribute check 
    11271129 
    1128 class Constraint__VarDimsCordexHardWired: 
     1130class Constraint__VarDimsCordexHardWired(object): 
    11291131  def __init__(self, attribVocabs,  kpat, keys, logger=None): 
    11301132     self.code = 'CQC.102.002.010' 
     
    11361138     self.plev_vars = ['clh','clm','cll','ua850','va850'] 
    11371139 
    1138 class Constraint__VarAtts: 
     1140class Constraint__VarAtts(object): 
    11391141 
    11401142  def __init__(self, attribVocabs,  kpat, keys, logger=None): 
     
    12021204 
    12031205#### check whether a NS element is constant 
    1204 class Constraint__Constant: 
     1206class Constraint__Constant(object): 
    12051207 
    12061208  def __init__(self, ref1, required=False): 
     
    12441246           return ('VALUE',bits[1]) 
    12451247 
    1246 class section_parser_l0: 
     1248class section_parser_l0(object): 
    12471249 
    12481250  def __init__(self,parent,sectionName): 
  • CCCC/trunk/ceda_cc/fcc_utils2.py

    r160 r180  
    11import string, os, re, stat, sys 
     2from config_c4 import CC_CONFIG_DIR 
    23 
    34ncdumpCmd = 'ncdump' 
     
    56## 
    67 
    7 class mipTableScan: 
     8from xceptions import * 
     9 
     10class mipTableScan(object): 
    811 
    912  def __init__(self, vats = ['standard_name','long_name','units','cell_methods'] ): 
     
    8689             log.warn(  'Mistake?? in scan_table %s' % str(mm) ) 
    8790          ds = mm 
    88           raise 'Mistake?? in scan_table %s' % str(mm) 
     91          raise baseException( 'Mistake?? in scan_table %s' % str(mm) ) 
    8992        ee.append( (var,ds,aa,tag) ) 
    9093 
     
    119122      else: 
    120123        return ff 
     124 
     125 
     126class snlist: 
     127 
     128  def __init__(self,dir=None,tab='cf-standard-name-table.xml' ): 
     129    if dir  == None: 
     130      dir = os.path.join(CC_CONFIG_DIR, 'cf/') 
     131    self.re_sn = re.compile( 'entry id="(.*)"' ) 
     132    self.re_sna = re.compile( 'alias id="(.*)"' ) 
     133    self.dir = dir 
     134    self.tab = tab 
     135 
     136##alias id="atmosphere_water_vapor_content" 
     137##entry id="age_of_sea_ice"' 
     138 
     139  def gen_sn_list(self ): 
     140    pathn = self.dir + self.tab 
     141    assert os.path.isfile( pathn ), '%s not found ' % pathn 
     142    snl = [] 
     143    snla = [] 
     144    for l in open(pathn).readlines(): 
     145      m = self.re_sn.findall(l ) 
     146      if len(m) > 0: 
     147        for i in m: 
     148          snl.append( i ) 
     149      m = self.re_sna.findall(l ) 
     150      if len(m) > 0: 
     151        for i in m: 
     152          snla.append( i ) 
     153    return (snl,snla) 
     154 
     155class tupsort: 
     156   def __init__(self,k=0): 
     157     self.k = k 
     158   def cmp(self,x,y): 
     159     return cmp( x[self.k], y[self.k] ) 
     160 
     161class tupsort2: 
     162   def __init__(self,k0,k1): 
     163     self.k0 = k0 
     164     self.k1 = k1 
     165   def cmp(self,x,y): 
     166     if x[self.k0] == y[self.k0]: 
     167       return cmp( x[self.k1], y[self.k1] ) 
     168     return cmp( x[self.k0], y[self.k0] ) 
     169 
     170class tupsort3: 
     171   def __init__(self,k0,k1,k2): 
     172     self.k0 = k0 
     173     self.k1 = k1 
     174     self.k2 = k2 
     175   def cmp(self,x,y): 
     176     if x[self.k0] == y[self.k0]: 
     177       if x[self.k1] == y[self.k1]: 
     178         return cmp( x[self.k2], y[self.k2] ) 
     179       return cmp( x[self.k1], y[self.k1] ) 
     180     return cmp( x[self.k0], y[self.k0] ) 
  • CCCC/trunk/ceda_cc/file_utils.py

    r160 r180  
    22# Standard library imports 
    33import string, pkgutil 
     4 
     5from xceptions import * 
    46 
    57# Third party imports 
     
    3739## end of netcdf import. 
    3840 
    39 class fileMetadata: 
     41class fileMetadata(object): 
    4042 
    4143  def __init__(self,dummy=False,attributeMappingsLog=None): 
     
    6668      self.loadNc__Scientific(fpath) 
    6769    else: 
    68       raise 'No supported netcdf module assigned' 
     70      raise baseException( 'No supported netcdf module assigned' ) 
    6971 
    7072  def loadNc__Cdms(self,fpath): 
  • CCCC/trunk/ceda_cc/summary.py

    r160 r180  
    22import string, sys, glob 
    33 
    4 args = sys.argv[1:-1] 
    5 idir = sys.argv[-1] 
    6 ndisp = 2 
    7 while len(args) > 0: 
    8   x = args.pop(0) 
    9   if x == '-n': 
    10     ndisp = int( args.pop(0) ) 
    11  
    12 fl = glob.glob( '%s/*__qclog_*.txt' % idir ) 
    13  
    14 ee = {} 
    15 print 'Summarising error reports from %s log file' % len(fl) 
    16 nne = 0 
    17 for f in fl: 
    18   nef = 0 
    19   for l in open(f).readlines(): 
    20     fn = string.split(f,'/')[-1] 
    21     if string.find(l, 'FAILED') != -1 or string.find(l,'CDMSError:') != -1: 
    22       nef += 1 
    23       if string.find(l, 'FAILED') != -1: 
    24          kb1 = 3 
    25       else: 
    26          kb1 = 1 
    27       bits = string.split(l, ':' ) 
    28       if len(bits) > kb1: 
    29         code = bits[0] 
    30         msg = string.strip( string.join(bits[kb1:], ':' ) ) 
    31         if code not in ee.keys(): 
    32           ee[code] = [0,{msg:[0,[]]}] 
    33         elif msg not in ee[code][1].keys(): 
    34           ee[code][1][msg] = [0,[]] 
    35         ee[code][0] += 1 
    36         ee[code][1][msg][0] += 1 
    37         if ee[code][1][msg][0] < 10: 
    38           ee[code][1][msg][1].append(fn) 
    39       else: 
    40         print bits 
    41   if nef == 0: 
    42     nne += 1 
    43  
    44 keys = ee.keys() 
    45 keys.sort() 
    464def cmin(x,y): 
    475  if x < 0: 
    48     return y 
     6        return y 
    497  else: 
    508    return min(x,y) 
    519 
    52 for k in keys: 
    53   ks = ee[k][1].keys() 
    54   if len(ks) == 1: 
    55     print k,ee[k][0],ks[0] 
    56     for i in range(cmin(ndisp,ee[k][0])): 
    57       print '               ',ee[k][1][ks[0]][1][i] 
    58   else: 
    59     print k,ee[k][0] 
    60     ks.sort() 
    61     for k2 in ks: 
    62       print '  --- ',k2,ee[k][1][k2][0] 
    63       for i in range(cmin(ndisp,ee[k][1][k2][0])): 
    64         print '               ',ee[k][1][k2][1][i] 
     10class main(object): 
     11 
     12  def __init__(self): 
     13    args = sys.argv[1:-1] 
     14    idir = sys.argv[-1] 
     15    ndisp = 2 
     16    while len(args) > 0: 
     17      x = args.pop(0) 
     18      if x == '-n': 
     19        ndisp = int( args.pop(0) ) 
     20 
     21    fl = glob.glob( '%s/*__qclog_*.txt' % idir ) 
     22 
     23    ee = {} 
     24    print 'Summarising error reports from %s log file' % len(fl) 
     25    nne = 0 
     26    for f in fl: 
     27      nef = 0 
     28      for l in open(f).readlines(): 
     29        fn = string.split(f,'/')[-1] 
     30        if string.find(l, 'FAILED') != -1 or string.find(l,'CDMSError:') != -1: 
     31          nef += 1 
     32          if string.find(l, 'FAILED') != -1: 
     33             kb1 = 3 
     34          else: 
     35             kb1 = 1 
     36          bits = string.split(l, ':' ) 
     37          if len(bits) > kb1: 
     38            code = bits[0] 
     39            msg = string.strip( string.join(bits[kb1:], ':' ) ) 
     40            if code not in ee.keys(): 
     41              ee[code] = [0,{msg:[0,[]]}] 
     42            elif msg not in ee[code][1].keys(): 
     43              ee[code][1][msg] = [0,[]] 
     44            ee[code][0] += 1 
     45            ee[code][1][msg][0] += 1 
     46            if ee[code][1][msg][0] < 10: 
     47              ee[code][1][msg][1].append(fn) 
     48          else: 
     49            print bits 
     50      if nef == 0: 
     51        nne += 1 
     52 
     53    keys = ee.keys() 
     54    keys.sort() 
    6555 
    6656 
    67 print 'Number of files with no errors: %s' % nne 
     57    for k in keys: 
     58      ks = ee[k][1].keys() 
     59      if len(ks) == 1: 
     60        print k,ee[k][0],ks[0] 
     61        for i in range(cmin(ndisp,ee[k][0])): 
     62          print '               ',ee[k][1][ks[0]][1][i] 
     63      else: 
     64        print k,ee[k][0] 
     65        ks.sort() 
     66        for k2 in ks: 
     67          print '  --- ',k2,ee[k][1][k2][0] 
     68          for i in range(cmin(ndisp,ee[k][1][k2][0])): 
     69            print '               ',ee[k][1][k2][1][i] 
    6870 
    6971 
     72    print 'Number of files with no errors: %s' % nne 
    7073 
     74if __name__ == '__main__': 
     75  main() 
  • CCCC/trunk/ceda_cc/unitTestsS1.py

    r162 r180  
    1616log.setLevel(logging.INFO) 
    1717 
    18 class dummy: 
     18class dummy(object): 
    1919 
    2020  def __init__(self): 
     
    3838c.check( fn ) 
    3939if c.errorCount == 0: 
    40   print 'Passed [%s] %s: valid file name' % (module,fn) 
     40  print 'OK: [%s] %s: valid file name' % (module,fn) 
    4141else: 
    4242  print 'Failed [%s] %s: valid file name' % (module,fn) 
     
    4646c.check(fn) 
    4747if c.errorCount == 0 and c.isFixed: 
    48   print 'Passed [%s] %s: valid file name' % (module,fn) 
     48  print 'OK: [%s] %s: valid file name' % (module,fn) 
    4949else: 
    5050  print 'Failed [%s] %s: valid file name' % (module,fn) 
     
    5454c.check(fn) 
    5555if c.errorCount == 0: 
    56   print 'Passed [%s] %s: valid file name' % (module,fn) 
     56  print 'OK: [%s] %s: valid file name' % (module,fn) 
    5757else: 
    5858  print 'Failed [%s] %s: valid file name' % (module,fn) 
     
    6262c.check(fn) 
    6363if c.errorCount == 0: 
    64   print 'Passed [%s] %s: valid file name' % (module,fn) 
     64  print 'OK: [%s] %s: valid file name' % (module,fn) 
    6565else: 
    6666  print 'Failed [%s] %s: valid file name' % (module,fn) 
     
    7272  print 'Failed to detect bad file name: [%s] %s ' % (module,fn) 
    7373else: 
    74   print 'OK -- detected bad file name: [%s] %s' % (module,fn) 
     74  print 'OK: -- detected bad file name: [%s] %s' % (module,fn) 
    7575 
    7676c = utils_c4.checkStandardDims(parent=p) 
     
    8080  print 'Failed [%s]: failed to detect empty dictionaries' % module 
    8181else: 
    82   print 'OK -- detected error in standard dims' 
     82  print 'OK: -- detected error in standard dims' 
    8383 
    8484c = utils_c4.checkByVar(parent=p) 
     
    9090  m = r1.match( x ) 
    9191  if m: 
    92      print 'OK -- passed [%s] %s for sub-daily data' % (module,x) 
     92     print 'OK: -- passed [%s] %s for sub-daily data' % (module,x) 
    9393  else: 
    9494     print 'Failed to match correct sub-daily time range element [%s] %s' % (module,x) 
     
    9797  m = r1.match( x ) 
    9898  if not m: 
    99      print 'OK -- correctly failed [%s] %s for sub-daily data' % (module,x) 
     99     print 'OK: -- correctly failed [%s] %s for sub-daily data' % (module,x) 
    100100  else: 
    101101     print 'Failed to detect bad sub-daily time range element [%s] %s' % (module,x) 
     
    105105  m = r1.match( x ) 
    106106  if m: 
    107      print 'OK -- passed [%s] %s for seasonal data' % (module,x) 
     107     print 'OK: -- passed [%s] %s for seasonal data' % (module,x) 
    108108  else: 
    109109     print 'Failed to match correct seasonal time range element [%s] %s' % (module,x) 
     
    112112  m = r1.match( x ) 
    113113  if not m: 
    114      print 'OK -- correctly failed [%s] %s for seasonal data' % (module,x) 
     114     print 'OK: -- correctly failed [%s] %s for seasonal data' % (module,x) 
    115115  else: 
    116116     print 'Failed to detect bad seasonal time range element [%s] %s' % (module,x) 
     
    120120  m = r1.match( x ) 
    121121  if m: 
    122      print 'OK -- passed [%s] %s for daily data' % (module,x) 
     122     print 'OK: -- passed [%s] %s for daily data' % (module,x) 
    123123  else: 
    124124     print 'Failed to match correct daily time range element [%s] %s -- %s' % (module,x,c.pats['day'][0]) 
     
    133133c.check( 'tas','AFR-44i', da, {'tas':{} } ) 
    134134if c.errorCount == 0: 
    135   print 'OK -- passed a correct grid' 
     135  print 'OK: -- passed a correct grid' 
    136136else: 
    137137  print 'Failed -- reported errors on correct grid' 
     
    143143  print 'Failed -- passed a bad grid' 
    144144else: 
    145   print 'OK -- detected a bad grid' 
     145  print 'OK: -- detected a bad grid' 
    146146 
    147147ii = open( op.join(config.CC_CONFIG_DIR, 'specs_vocabs/globalAtsSample001.txt') ) 
     
    153153c.check( fn ) 
    154154if c.errorCount == 0: 
    155   print 'Passed [%s] %s: valid file name' % (module,fn) 
     155  print 'OK: [%s] %s: valid file name' % (module,fn) 
    156156else: 
    157157  print 'Failed [%s] %s: valid file name' % (module,fn) 
     
    180180c.check( fn ) 
    181181if c.errorCount == 0: 
    182   print 'Passed [%s] %s: valid file name' % (module,fn) 
     182  print 'OK: [%s] %s: valid file name' % (module,fn) 
    183183else: 
    184184  print 'Failed [%s] %s: valid file name' % (module,fn) 
  • CCCC/trunk/ceda_cc/unitTestsS2.py

    r160 r180  
    44import config_c4 as config 
    55from c4 import fileMetadata, dummy, main 
     6from xceptions import * 
    67 
    78#### set up log file #### 
     
    2021except: 
    2122  print 'Failed to parse a simple dummy file path' 
    22   raise 
     23  raise baseException( 'Failed to parse a simple dummy file path' ) 
    2324print 'OK: instantiated fileMetaData and parsed a simple dummy path' 
    2425 
     
    4748except: 
    4849  print 'Failed [%s]: dummy run triggered exception' % testId 
    49   raise 
     50  raise baseException( 'Failed [%s]: dummy run triggered exception' % testId ) 
    5051 
    5152testId = '#11.002' 
  • CCCC/trunk/ceda_cc/utils_c4.py

    r173 r180  
    1  
    21import string, re, os, sys, traceback 
    32 
     
    65 
    76from fcc_utils import mipTableScan 
    8  
    9 class reportSection: 
     7from xceptions import * 
     8 
     9class reportSection(object): 
    1010 
    1111  def __init__(self,id,cls,parent=None, description=None): 
     
    5555        self.fail += 1 
    5656 
    57 class abortChecks(Exception): 
    58   pass 
    59 class loggedException(Exception): 
    60   pass 
    61 class baseException(Exception): 
    62   
    63   def __init__(self,msg): 
    64     self.msg = 'utils_c4:: %s' % msg 
    65  
    66   def __str__(self): 
    67         return unicode(self).encode('utf-8') 
    68  
    69   def __unicode__(self): 
    70         return self.msg % tuple([force_unicode(p, errors='replace') 
    71                                  for p in self.params]) 
    72 class checkSeq: 
     57class checkSeq(object): 
    7358  def __init__(self): 
    7459    pass 
     
    8368cs = checkSeq() 
    8469 
    85 class checkBase: 
     70class checkBase(object): 
    8671 
    8772  def  __init__(self,cls="CORDEX",reportPass=True,parent=None,monitor=None): 
     
    462447        except: 
    463448          print 'failed trying to check global attribute %s' % a 
    464           raise 
     449          raise baseException( 'failed trying to check global attribute %s' % a ) 
    465450 
    466451    if not self.test( len(m)  == 0, 'Global attributes do not match constraints: %s' % str(m) ): 
     
    759744        self.log_pass() 
    760745 
    761 class mipVocab: 
     746class mipVocab(object): 
    762747 
    763748  def __init__(self,pcfg,dummy=False): 
     
    827812       return self.varInfo[k]['ac'] 
    828813     else: 
    829        raise 'mipVocab.lists called with bad list specifier %s' % k2 
     814       raise baseException( 'mipVocab.lists called with bad list specifier %s' % k2 ) 
    830815 
    831816  def isInTable( self, v, vg ): 
     
    839824    return self.varcons[vg][v][a] 
    840825       
    841 class patternControl: 
     826class patternControl(object): 
    842827 
    843828  def __init__(self,tag,pattern,list=None): 
     
    865850      return m.groupdict()["val"] in self.list 
    866851     
    867 class listControl: 
     852class listControl(object): 
    868853  def __init__(self,tag,list,split=False,splitVal=None): 
    869854    self.list = list 
     
    1002987 
    1003988### http://stackoverflow.com/questions/2023608/check-what-files-are-open-in-python 
    1004 class sysMonitor: 
     989class sysMonitor(object): 
    1005990 
    1006991  def __init__(self): 
Note: See TracChangeset for help on using the changeset viewer.