| 1 | """Import data request from shelves and pu onto compliant XML document. |
|---|
| 2 | ------------------------------------------------------------------------ |
|---|
| 3 | A new uuid is generated for elements of the "revisedtabitem" section/table (because it is not present in the shelve). |
|---|
| 4 | |
|---|
| 5 | NOTES |
|---|
| 6 | ----- |
|---|
| 7 | July 27th: there is an unresolved problem about preserving uuids when updates are provided in source format ..... |
|---|
| 8 | """ |
|---|
| 9 | |
|---|
| 10 | ### Experiment group dictionary added, and experiment set of named tuples -- making it possible to identify experiments in each mip and group. |
|---|
| 11 | ### Should be able to use this in review to make reference groups ... |
|---|
| 12 | |
|---|
| 13 | ### |
|---|
| 14 | ### Appear to be getting core variables into a clean format |
|---|
| 15 | ### Some filtering to avoit duplicates arising from two approaches to OMIP, flagged as ERROR.090 in standard output. |
|---|
| 16 | ### Corrected by omitting 2nd reference |
|---|
| 17 | ### |
|---|
| 18 | |
|---|
| 19 | import xlrd, xml, os, sys, string, shelve, json |
|---|
| 20 | import utils_wb, uuid |
|---|
| 21 | import xml.dom, xml.dom.minidom |
|---|
| 22 | import collections, string, re |
|---|
| 23 | import choiceCfg |
|---|
| 24 | import importWbMods |
|---|
| 25 | import dreq_utils |
|---|
| 26 | from utils_wb import uniCleanFunc |
|---|
| 27 | import ivg |
|---|
| 28 | |
|---|
| 29 | mipTabMaps = json.load( open('inputs/mipTableMaps.txt', 'r' ) ) |
|---|
| 30 | |
|---|
| 31 | #up: |
|---|
| 32 | varPositive = {'rlus':'up', 'rlut':'up', 'rlutcs':'up', 'hfls':'up', 'hfss':'up', 'mcu':'up', 'rsus':'up', 'rsut':'up', 'rsutcs':'up', 'rlds':'down', 'rsds':'down' } |
|---|
| 33 | |
|---|
| 34 | refMips = ['AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] |
|---|
| 35 | ## |
|---|
| 36 | ## refCmv.refUid[tab][lab] = k |
|---|
| 37 | ## |
|---|
| 38 | refCmv = ivg.refCmv() |
|---|
| 39 | extRefCmvMode = 'r' |
|---|
| 40 | extRefCmv = shelve.open( 'sh/extRefCmv' ) |
|---|
| 41 | extRefCmvInv = {} |
|---|
| 42 | nd = 0 |
|---|
| 43 | sd = set() |
|---|
| 44 | sd2 = set() |
|---|
| 45 | for k in extRefCmv: |
|---|
| 46 | k1 = extRefCmv[k] |
|---|
| 47 | tab,var = k.split( '.' ) |
|---|
| 48 | if k1 in extRefCmvInv: |
|---|
| 49 | tab2, var2 = extRefCmvInv[k1].split('.') |
|---|
| 50 | if var == var2 and (tab == mipTabMaps.get( tab2, None) or tab2 == mipTabMaps.get( tab, None) ): |
|---|
| 51 | pass |
|---|
| 52 | else: |
|---|
| 53 | print 'FATAL: Duplicate key in extRefCmv: %s [%s] %s' % (k,k1,extRefCmvInv[k1]) |
|---|
| 54 | sd.add(k) |
|---|
| 55 | sd2.add(extRefCmvInv[k1]) |
|---|
| 56 | nd += 1 |
|---|
| 57 | extRefCmvInv[k1] = k |
|---|
| 58 | assert nd == 0, 'FATAL ERRORS FOUND ... see standard output: %s, %s' % (str( sd ), str( sd2 ) ) |
|---|
| 59 | |
|---|
| 60 | extRefCmvMapped = {} |
|---|
| 61 | def chkRefCmv( ntab, lab, u0, msg ): |
|---|
| 62 | useExtRef = True |
|---|
| 63 | thattab = mipTabMaps.get( ntab, ntab ) |
|---|
| 64 | thistab = ntab |
|---|
| 65 | if thistab in refCmv.refUid and lab in refCmv.refUid[thistab]: |
|---|
| 66 | uid = refCmv.refUid[thistab][lab] |
|---|
| 67 | print 'INFO.uid0.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,uid) |
|---|
| 68 | if uid != u0: |
|---|
| 69 | extRefCmvMapped[u0] = uid |
|---|
| 70 | elif thattab in refCmv.refUid and lab in refCmv.refUid[thattab]: |
|---|
| 71 | uid = refCmv.refUid[thattab][lab] |
|---|
| 72 | print 'INFO.uid1.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,uid) |
|---|
| 73 | if uid != u0: |
|---|
| 74 | extRefCmvMapped[u0] = uid |
|---|
| 75 | else: |
|---|
| 76 | kk = str( '%s.%s' % (thistab,lab) ) |
|---|
| 77 | kkk = str( '%s.%s' % (thattab,lab) ) |
|---|
| 78 | if kk in extRefCmv: |
|---|
| 79 | if useExtRef: |
|---|
| 80 | uid = extRefCmv[kk] |
|---|
| 81 | print 'INFO.uid.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,uid) |
|---|
| 82 | if uid != u0: |
|---|
| 83 | extRefCmvMapped[u0] = uid |
|---|
| 84 | else: |
|---|
| 85 | u1 = extRefCmv[kk] |
|---|
| 86 | print 'INFO.uid.%s: uid in extRefCmv ignored: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,u1) |
|---|
| 87 | uid = u0 |
|---|
| 88 | elif kkk in extRefCmv: |
|---|
| 89 | if useExtRef: |
|---|
| 90 | uid = extRefCmv[kkk] |
|---|
| 91 | print 'INFO.uid4.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,uid) |
|---|
| 92 | if uid != u0: |
|---|
| 93 | extRefCmvMapped[u0] = uid |
|---|
| 94 | else: |
|---|
| 95 | u1 = extRefCmv[kkk] |
|---|
| 96 | print 'INFO.uid4.%s: uid in extRefCmv ignored: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,u1) |
|---|
| 97 | uid = u0 |
|---|
| 98 | else: |
|---|
| 99 | if extRefCmvMode != 'r': |
|---|
| 100 | extRefCmv[kk] = u0 |
|---|
| 101 | print 'WARN.uid.%s: uid not found: %s, %s (%s)' % (msg,thistab,lab,u0) |
|---|
| 102 | uid = u0 |
|---|
| 103 | return uid |
|---|
| 104 | |
|---|
| 105 | |
|---|
| 106 | siSurfFluxes = ['sidmasstranx', 'sidmasstrany', 'siforcecoriolx', 'siforcecorioly', 'siforcetiltx', 'siforcetilty', 'siforceintstrx', 'siforceintstry', 'sistrxdtop', 'sistrydtop', 'sistrxdbot', 'sistrydbot', 'siu', 'siv'] |
|---|
| 107 | oceanVars = ['fg13co2','fg14co2'] |
|---|
| 108 | realmTab = {'landice':['LImongre','LImonant','LIyrgre','LIyrant','LIfxgre','LIfxant','ImonGre','ImonAnt','IyrGre','IyrAnt','IfxGre','IfxAnt'], |
|---|
| 109 | 'atmos':['6hrLev','6hrPlev','6hrPevpt','em1hrclimmon','em3hr','em1hr','em3hrpt','emDayZ','emDaypt','emMonZ']} |
|---|
| 110 | ## emYr, emMon, emFy, emMonZ .... need checking |
|---|
| 111 | knownRealmA = [ 'LS3MIP [LWday]', 'LS3MIP [LEday]', 'CFMIP [cf1hrClimMon]', 'HighResMIP [3hr_cloud]', 'CFMIP [cf3hr_sim_new]', 'C4MIP [L_3hr]', 'DAMIP [DAMIP_day]', 'DAMIP [DAMIP_3hr_p2]', 'DynVar [DYVR_daily_c]', 'PMIP [PMIP-6hr]', 'HighResMIP [1hrLev]'] |
|---|
| 112 | knownRealmO = [ 'DAMIP [DAMIP_Omon_p2]', 'FAFMIP [fafOyr]'] |
|---|
| 113 | varRealmMaps = {'land':['areacellg','orog'], |
|---|
| 114 | 'landIce land':['snc'], |
|---|
| 115 | 'landIce':['icem', 'sftflf', 'sftgrf','acabf','snicefreez', 'snicem', 'hfgeoubed', 'lithk', 'topg','areacelli','lialb', 'libmassbffl', 'libmassbfgr', 'licalvf', 'lifmassbf', 'litempbotfl', 'litempbotgr', 'litemptop', 'mrroIs','strbasemag', 'topg', 'xvelbase', 'xvelmean', 'xvelsurf', 'yvelbase', 'yvelmean', 'yvelsurf', 'zvelbase', 'zvelsurf','snmIs','sncIs','tsnIs','sbl'], |
|---|
| 116 | 'atmos':['prra','prsn','prrc','sbl','tas','prCrop', 'tasmaxCrop', 'tasminCrop','hfls', 'hfss','rlds', 'rlus', 'rsds', 'rsus'] } |
|---|
| 117 | varRealmMapsInv = {} |
|---|
| 118 | for k in varRealmMaps: |
|---|
| 119 | for v in varRealmMaps[k]: |
|---|
| 120 | varRealmMapsInv[ v ] = k |
|---|
| 121 | |
|---|
| 122 | class replaceItems(object): |
|---|
| 123 | def __init__(self): |
|---|
| 124 | self.v = collections.defaultdict( list ) |
|---|
| 125 | self.items = {} |
|---|
| 126 | |
|---|
| 127 | WbMods = importWbMods.run() |
|---|
| 128 | tableUtils = ivg.tableUtils() |
|---|
| 129 | |
|---|
| 130 | nt__trace = collections.namedtuple( 'trace', ['GROUPITEM_ta','SPATIALSHAPE_01'] ) |
|---|
| 131 | trace = nt__trace( False, False ) |
|---|
| 132 | |
|---|
| 133 | tab2freq = {u'CMIP5_cfOff':'subhr', u'CORDEX_mon':'mon', u'SPECS_day':'day', u'CMIP5_day':'day', \ |
|---|
| 134 | u'PMIP3_OImon':'mon', u'CORDEX_day':'day', u'CMIP5_LImon':'mon', u'CMIP5_OImon':'mon', \ |
|---|
| 135 | u'CMIP5_Lmon':'mon', u'CMIP5_3hr':'3hr', u'CMIP5_Omon':'mon', u'PMIP3_OIclim':'monClim', \ |
|---|
| 136 | u'PMIP3_fx':'fx', u'CORDEX_fx':'fx', u'PMIP3_LImon':'mon', u'CMIP5_6hrPlev':'6hr', u'PMIP3_Lmon':'mon', \ |
|---|
| 137 | u'PMIP3_Amon':'mon', u'SPECS_Omon':'mon', u'CCMI1_fixed':'fx', u'PMIP3_Aclim':'monClim', u'CMIP5_6hrLev':'6hr', \ |
|---|
| 138 | u'CMIP5_Oclim':'monClim', u'PMIP3_LIclim':'monClim', u'CCMI1_monthly':'mon', u'CMIP5_fx':'fx', \ |
|---|
| 139 | u'CMIP5_cfDay':'day', u'CORDEX_6h':'6hr', u'PMIP3_day':'day', u'SPECS_OImon':'mon', u'CMIP5_cfMon':'mon', \ |
|---|
| 140 | u'CORDEX_sem':'monClim', u'SPECS_6hr':'6hr', u'CMIP5_cfSites':'subhr', u'CCMI1_hourly':'hr', u'CMIP5_aero':'mon', \ |
|---|
| 141 | u'CMIP5_Amon':'mon', u'PMIP3_Omon':'mon', u'CCMI1_daily':'day', u'SPECS_fx':'fx', u'PMIP3_Lclim':'monClim', \ |
|---|
| 142 | u'aermonthly':'mon', u'aermonthlyz':'mon', u'aerhourly':'hr', 'aerfixed':'fx', 'aerdaily':'day', |
|---|
| 143 | u'DCPP-day':'day', \ |
|---|
| 144 | u'Odec':'dec', \ |
|---|
| 145 | u'PMIP3_Oclim':'monClim', u'SPECS_Amon':'mon', u'SPECS_Lmon':'mon', u'CMIP5_cf3hr':'3hr', u'CORDEX_3h':'3hr', \ |
|---|
| 146 | 'OImon':'mon', 'Omon_oth':'mon', 'Omon':'mon', 'Oyr':'yr', 'Omon_3d':'mon', \ |
|---|
| 147 | u'CCMI1_annual':'yr', u'CMIP5_Oyr':'yr', \ |
|---|
| 148 | 'Oclim':'monClim', \ |
|---|
| 149 | 'cfMon':'mon', 'emMonZ':'mon', \ |
|---|
| 150 | '3hr':'3hr', 'cfOff':'subhr', 'OMIP-Omon':'mon', '6hrLev':'6hr', 'fx':'fx', 'OMIP-Oyr':'yr', 'cfDay':'day', 'Lmon':'day', \ |
|---|
| 151 | u'cf3hr':'3hr', u'Amon':'mon', u'aero':'mon', u'aero_3d':'mon', u'6hrPlev':'6hr', u'aero_oth':'mon', \ |
|---|
| 152 | u'cf3hr_grid':'3hr', u'LImon':'mon', u'cfSites':'subhr', u'day_oth':'day', u'cfDay_2d':'day', u'day':'day' } |
|---|
| 153 | |
|---|
| 154 | freq2tab = collections.defaultdict( list ) |
|---|
| 155 | for k in tab2freq: |
|---|
| 156 | freq2tab[tab2freq[k]].append( k ) |
|---|
| 157 | |
|---|
| 158 | tableNameMap = {'CMIP5_3hr':'3hr', 'CMIP5_6hrLev':'6hrLev', 'CMIP5_6hrPlev':'6hrPlev', 'CMIP5_Amon':'Amon', 'CMIP5_LImon':'LImon', 'CMIP5_Lmon':'Lmon', 'CMIP5_OImon':'CMIP5_OImon', 'CMIP5_Oclim':'Oclim', 'CMIP5_Omon':'CMIP5_Omon', 'CMIP5_aero':'aero', 'CMIP5_cf3hr':'cf3hr', 'CMIP5_cfDay':'cfDay', 'CMIP5_cfMon':'cfMon', 'CMIP5_cfOff':'cfOff', 'CMIP5_cfSites':'cfsites', 'CMIP5_day':'day', 'CMIP5_fx':'fx', 'OMIP.Omon':'Omon', 'OMIP.Oyr':'Oyr', 'OMIP.day':'Oday', 'OMIP.fx':'Ofx' } |
|---|
| 159 | |
|---|
| 160 | reportedRedundantTables = set() |
|---|
| 161 | redundantTables = ['aero','CMIP5_Oyr','CMIP5_Omon','CMIP5_OImon','CORDEX_3h', 'CORDEX_6h', 'CORDEX_fx', 'CORDEX_mon', 'CORDEX_sem', 'PMIP3_Aclim', 'PMIP3_Amon', 'PMIP3_LIclim', 'PMIP3_LImon', 'PMIP3_Lclim', 'PMIP3_Lmon', 'PMIP3_OIclim', 'PMIP3_OImon', 'PMIP3_Oclim', 'PMIP3_Omon', 'PMIP3_day', 'PMIP3_fx', |
|---|
| 162 | 'SPECS_Lmon', 'SPECS_Amon', 'SPECS_6hr', 'SPECS_day', 'SPECS_fx', 'SPECS_Omon', 'SPECS_OImon', |
|---|
| 163 | 'CORDEX_day', |
|---|
| 164 | 'CCMI1_annual','CCMI1_daily','CCMI1_fixed','CCMI1_hourly','CCMI1_monthly'] |
|---|
| 165 | |
|---|
| 166 | ## mapping of variable groups representing tables .... |
|---|
| 167 | mappedTables = {'Omon':'OMIP-Omon', 'OMIP-Oyr':'Oyr', 'OImon':'SIMIP-seaicemon' } |
|---|
| 168 | mappedTables = {'Omon':'OMIP-Omon', 'CMIP5-Oyr':'OMIP-Oyr', 'OImon':'SIMIP-seaicemon', 'CMIP5-Omon':'OMIP-Omon' } |
|---|
| 169 | ## mappedTables = { 'OImon':'SIMIP-seaicemon' } |
|---|
| 170 | ## 'aeromonthly':'aermonthly' } |
|---|
| 171 | |
|---|
| 172 | freqmap = {'daily':'day', 'Annual':'yr', 'Timestep':'subhr', '1day':'day', '1mon':'mon', 'month':'mon', 'year':'yr', 'monthly':'mon', 'Day':'day', '6h':'6hr', '3 hourly':'3hr', '3 Hourly':'3hr' } |
|---|
| 173 | |
|---|
| 174 | nt_expt = collections.namedtuple( 'expt', ['uid', 'label', 'mip', 'egid' ] ) |
|---|
| 175 | realmdef = { '__all__':['CFMIP [cfMonExtra]', 'PMIP [PMIP-Omon]', 'LS3MIP [LCmon]', 'C4MIP [C_Ocean_T2]', 'C4MIP [C_Ocean_T1]', 'LUMIP [Lyr_Lut]', 'PMIP [PMIP-LIclim]', 'HighResMIP [1hr_strat]', 'SIMIP [seaiceday]', 'C4MIP [C_LandT1]', 'CFMIP [cfDay_2d_new]', 'HighResMIP [6hrPlev_intense]', 'HighResMIP [Amon_diag]', 'DCPP [DCPP-mon]', 'HighResMIP [Amon_conv]', 'SIMIP [seaicemon]', 'DAMIP [new_monthly]', 'FAFMIP [fafOyrB]', 'HighResMIP [Amon_ext]', 'DynVar [DYVR_daily]', 'PMIP [PMIP-Aclim]', 'HighResMIP [3hr_extr]', 'C4MIP [C_Basic]', 'DCPP [DCPP-day]', 'CFMIP [cfMon_3dstd_new]', 'FAFMIP [fafOmon]', 'ISMIP6 [new_fx]', 'LS3MIP [LEday]', 'PMIP [PMIP-aeroclim]', 'PMIP [PMIP-aero]', 'DCPP [DCPP-6hr]', 'DynVar [DYVR_monthly]', 'ISMIP6 [icesheetmon]', 'CFMIP [aeroDay_2d]', 'ISMIP6 [new_LImon]', 'ISMIP6 [new_Omon]', 'CFMIP [cfDayExtra]', 'LS3MIP [LWday]', 'PMIP [PMIP-Oclim]', 'ISMIP6 [icesheetyear]', 'LUMIP [Lmon_Lut]', 'HighResMIP [1ts]', 'PMIP [PMIP-OIclim]', 'HighResMIP [6hrPlev_extr_dr]', 'VolMIP [VIRF]', 'C4MIP [C_LandT2]', 'C4MIP [C_Hist]', 'FAFMIP [fafOmonB]', 'RFMIP [aero_irf]', 'PMIP [PMIP-Lmon]', 'C4MIP [L_day]', 'PMIP [PMIP-Lclim]', 'PMIP [PMIP-Amon]', 'GeoMIP [aeroGeo]', 'HighResMIP [6hrPlev_extr]', 'PMIP [PMIP-day]', 'HighResMIP [Amon_sparc]', 'CFMIP [cfSites_new]', 'CFMIP [cf3hr_sim]'], |
|---|
| 176 | 'atmos':['CFMIP [cfMonExtra]', 'HighResMIP [1hr_strat]', 'CFMIP [cfDay_2d_new]', 'HighResMIP [6hrPlev_intense]', 'HighResMIP [Amon_diag]', 'DCPP [DCPP-mon]', 'HighResMIP [Amon_conv]', 'DAMIP [new_monthly]', 'HighResMIP [Amon_ext]', 'DynVar [DYVR_daily]', 'PMIP [PMIP-Aclim]', 'HighResMIP [3hr_extr]', 'DCPP [DCPP-day]', 'CFMIP [cfMon_3dstd_new]', 'PMIP [PMIP-aeroclim]', 'PMIP [PMIP-aero]', 'DCPP [DCPP-6hr]', 'DynVar [DYVR_monthly]', 'CFMIP [aeroDay_2d]', 'CFMIP [cfDayExtra]', 'HighResMIP [1ts]', 'HighResMIP [6hrPlev_extr_dr]', 'VolMIP [VIRF]', 'RFMIP [aero_irf]', 'PMIP [PMIP-Amon]', 'GeoMIP [aeroGeo]', 'HighResMIP [6hrPlev_extr]', 'PMIP [PMIP-day]', 'HighResMIP [Amon_sparc]', 'CFMIP [cfSites_new]', 'CFMIP [cf3hr_sim]'], |
|---|
| 177 | 'ocean':['PMIP [PMIP-Omon]', 'C4MIP [C_Ocean_T1]', 'C4MIP [C_Ocean_T2]', 'FAFMIP [fafOyrB]', 'FAFMIP [fafOmon]', 'ISMIP6 [new_Omon]', 'PMIP [PMIP-OIclim]', 'FAFMIP [fafOmonB]', 'PMIP [PMIP-Oclim]'], |
|---|
| 178 | 'land':['LS3MIP [LCmon]', 'C4MIP [C_Hist]', 'C4MIP [L_day]', 'PMIP [PMIP-Lclim]', 'C4MIP [C_Basic]', 'LUMIP [Lmon_Lut]', 'LUMIP [Lyr_Lut]', 'C4MIP [C_LandT1]', 'PMIP [PMIP-Lmon]', 'C4MIP [C_LandT2]'], |
|---|
| 179 | 'seaIce':['SIMIP [seaicemon]', 'SIMIP [seaiceday]' ], |
|---|
| 180 | 'landIce':['ISMIP6 [new_LImon]', 'PMIP [PMIP-LIclim]', 'ISMIP6 [icesheetyear]', 'ISMIP6 [new_fx]', 'ISMIP6 [icesheetmon]'], |
|---|
| 181 | '__unset__':['LS3MIP [LWday]', 'LS3MIP [LEday]'] } |
|---|
| 182 | |
|---|
| 183 | realmdefix = {} |
|---|
| 184 | for k in realmdef.keys(): |
|---|
| 185 | if k != '__all__': |
|---|
| 186 | for s in realmdef[k]: |
|---|
| 187 | assert not realmdefix.has_key(s), 'Duplicate targ value in realmdef: %s [%s]' % (s,k) |
|---|
| 188 | realmdefix[s] = k |
|---|
| 189 | |
|---|
| 190 | for k in realmdef['__all__']: |
|---|
| 191 | assert realmdefix.has_key(k), 'Key %s not found in realmdefix' % k |
|---|
| 192 | |
|---|
| 193 | class cls_s1(object): |
|---|
| 194 | def __init__(self): |
|---|
| 195 | self.a = collections.defaultdict( list ) |
|---|
| 196 | |
|---|
| 197 | def mycc(s): |
|---|
| 198 | s = string.replace( s, '&', '' ) |
|---|
| 199 | s = string.replace( s, '.', '-' ) |
|---|
| 200 | if len(s) > 0 and s[-1] == '+': |
|---|
| 201 | s = s[:-1] + 'ETC' |
|---|
| 202 | ## camelcase the input string |
|---|
| 203 | s1 = string.lower( string.replace(string.strip(str( s) ), '-', ' ') ) |
|---|
| 204 | return string.replace( string.capwords( s1 ), ' ', '' ) |
|---|
| 205 | |
|---|
| 206 | empty=re.compile('^$') |
|---|
| 207 | |
|---|
| 208 | def test( x,m): |
|---|
| 209 | if not x: |
|---|
| 210 | print m |
|---|
| 211 | return x |
|---|
| 212 | |
|---|
| 213 | class lcm(object): |
|---|
| 214 | def __init__(self,a,b): |
|---|
| 215 | self.a = {} |
|---|
| 216 | self.b = {} |
|---|
| 217 | for i in a: |
|---|
| 218 | self.a[string.lower(i)] = i |
|---|
| 219 | for i in b: |
|---|
| 220 | self.b[string.lower(i)] = i |
|---|
| 221 | |
|---|
| 222 | class main(object): |
|---|
| 223 | |
|---|
| 224 | def __init__(self,src,rq,doRepl=False,run=False,schemaMode='dreq'): |
|---|
| 225 | self.schemaMode = schemaMode |
|---|
| 226 | self.requestlinkuid = set() |
|---|
| 227 | self.src=src |
|---|
| 228 | self.vgcheck = {} |
|---|
| 229 | self.rqlPreset = {} |
|---|
| 230 | fok = [test(os.path.isfile(src),'%s not found' % src), ] |
|---|
| 231 | assert all( fok), 'Required input file(s) missing' |
|---|
| 232 | self.fix = WbMods.fix |
|---|
| 233 | self.spsh = WbMods.ls |
|---|
| 234 | self.spsh.addTime() |
|---|
| 235 | self.cmDims = WbMods.cm |
|---|
| 236 | self.cmvMap002 = dict() |
|---|
| 237 | self.importWbMods = WbMods |
|---|
| 238 | self.rq = rq |
|---|
| 239 | self.deferredLabMap = {} |
|---|
| 240 | self.struidmapextra = {} |
|---|
| 241 | |
|---|
| 242 | ## groupItems:grps |
|---|
| 243 | for sh in [rq.expt,rq.exgp,rq.objec,rq.refti, rq.revti, rq.grps, rq.rqli, rq.rqit]: |
|---|
| 244 | print sh['__info__'] |
|---|
| 245 | |
|---|
| 246 | self.repl = collections.defaultdict( list ) |
|---|
| 247 | self.replItems = {} |
|---|
| 248 | self.rep = {'cmv':replaceItems()} |
|---|
| 249 | ##self.deferredRequestLinkClone = collections.defaultdict( set ) |
|---|
| 250 | |
|---|
| 251 | self.err0010 = collections.defaultdict( int ) |
|---|
| 252 | self.defaultP = {} |
|---|
| 253 | self.remo = {} |
|---|
| 254 | self.upda = {} |
|---|
| 255 | self.insert = {} |
|---|
| 256 | self.rqLnks = collections.OrderedDict() |
|---|
| 257 | self.exptSet = set() |
|---|
| 258 | self.exptGrpDict = {} |
|---|
| 259 | |
|---|
| 260 | self.objectives = collections.defaultdict( dict ) |
|---|
| 261 | self.objectiveLinks = collections.defaultdict( cls_s1 ) |
|---|
| 262 | self.experiments = {} |
|---|
| 263 | self.experimentGrp = {} |
|---|
| 264 | self.exptPntUid = {} |
|---|
| 265 | self.exptPnt = collections.defaultdict( list ) |
|---|
| 266 | self.mips = set() |
|---|
| 267 | self.dsortdd = {} |
|---|
| 268 | |
|---|
| 269 | if doRepl: |
|---|
| 270 | self.importRepl() |
|---|
| 271 | self.importRepl(rfile='CMVreplace.csv',mode='cmv') |
|---|
| 272 | self.importRemove() |
|---|
| 273 | self.importUpdate() |
|---|
| 274 | self.importInsert() |
|---|
| 275 | |
|---|
| 276 | self.e15_10 = 0 |
|---|
| 277 | self.doc = xml.dom.minidom.parse( self.src ) |
|---|
| 278 | self.vocabs = xml.dom.minidom.parse( 'vocabs.xml' ) |
|---|
| 279 | self.skey = {'experiment':rq.expt, 'exptgroup':rq.exgp, \ |
|---|
| 280 | 'objective':rq.objec, 'var':None,'ovar':rq.refti, 'groupitem':rq.grps, \ |
|---|
| 281 | 'revisedtabitem_xxx':rq.revti, 'requestlink':rq.rqli, 'requestitem':rq.rqit, \ |
|---|
| 282 | 'requestvargroup':rq.rqvg, 'tablesection':rq.rqsect, 'requestvar':rq.revti} |
|---|
| 283 | |
|---|
| 284 | self.newImport = False |
|---|
| 285 | self.prepVar() |
|---|
| 286 | self.writeTimeSlice() |
|---|
| 287 | self.writeMip() |
|---|
| 288 | self.writeMcfg() |
|---|
| 289 | self.writeProcNotes() |
|---|
| 290 | self.writeCmDim() |
|---|
| 291 | self.writeRvg() |
|---|
| 292 | self.prep(rq) |
|---|
| 293 | self.cmvAudit(0) |
|---|
| 294 | self.prepRequestItem() |
|---|
| 295 | self.reviewExpt() |
|---|
| 296 | if run: |
|---|
| 297 | self.run(rq) |
|---|
| 298 | self.writeRelations() |
|---|
| 299 | self.writeRvx() |
|---|
| 300 | if self.schemaMode == 'dreq2': |
|---|
| 301 | self.write2() |
|---|
| 302 | self.writeMipTables() |
|---|
| 303 | self.writeCellMethods() |
|---|
| 304 | self.writeVar() |
|---|
| 305 | self.writeSn() |
|---|
| 306 | self.finish() |
|---|
| 307 | self.finish2() |
|---|
| 308 | extRefCmv.close() |
|---|
| 309 | |
|---|
| 310 | def prepRequestItem(self): |
|---|
| 311 | thissh = self.skey['requestitem'] |
|---|
| 312 | ## 'mip', 'tab', 'expt','rlid','ny', 'nexmax', 'nenmax', 'nymax','treset','info' |
|---|
| 313 | s1 = collections.defaultdict( set ) |
|---|
| 314 | for k in thissh: |
|---|
| 315 | if k[0] != '_': |
|---|
| 316 | ll = thissh[k][:] |
|---|
| 317 | ku = str(ll[2]) |
|---|
| 318 | s1[ll[0]].add(ll[9]) |
|---|
| 319 | if len( ku) == 1: |
|---|
| 320 | print 'SEVERE.rqi.00088: ',ll |
|---|
| 321 | ## |
|---|
| 322 | self.exptPnt[ku].append( str(k) ) |
|---|
| 323 | |
|---|
| 324 | thissh = self.skey['experiment'] |
|---|
| 325 | for k in thissh: |
|---|
| 326 | if k[0] != '_': |
|---|
| 327 | ll = thissh[k][:] |
|---|
| 328 | thisl = str( ll[2] ) |
|---|
| 329 | if ll[4] in ['SolarMIP']: |
|---|
| 330 | print 'ERROR: %s experiment: %s' % (ll[4],str(ll)) |
|---|
| 331 | else: |
|---|
| 332 | self.mips.add( str( ll[4] ) ) |
|---|
| 333 | self.experiments[thisl] = str( ll[0] ) |
|---|
| 334 | self.exptSet.add( nt_expt._make( [str( ll[0] ), thisl, str( ll[4] ), str( ll[1] ) ] ) ) |
|---|
| 335 | |
|---|
| 336 | thissh = self.skey['exptgroup'] |
|---|
| 337 | for k in thissh: |
|---|
| 338 | if k[0] != '_': |
|---|
| 339 | ll = thissh[k][:] |
|---|
| 340 | thisl = str( ll[1] ) |
|---|
| 341 | self.exptGrpDict[thisl] = str( ll[1] ) |
|---|
| 342 | self.experimentGrp[thisl] = str( ll[0] ) |
|---|
| 343 | |
|---|
| 344 | mm = [] |
|---|
| 345 | for m in self.mips: |
|---|
| 346 | if m not in self.mipUids: |
|---|
| 347 | mm.append(m) |
|---|
| 348 | assert len(mm) == 0, 'Not all mips found: %s: %s' % (str(mm),str(self.mipUids)) |
|---|
| 349 | |
|---|
| 350 | def writeProcNotes(self): |
|---|
| 351 | xx = self.vocabs.getElementsByTagName( 'processingOptions' ) |
|---|
| 352 | assert len(xx) == 1, 'Expecting one element named "processingOptions", found %s' % len(xx) |
|---|
| 353 | iDoc = xx[0] |
|---|
| 354 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
|---|
| 355 | thiss = self.doc.getElementsByTagName( 'tags' )[0] |
|---|
| 356 | dil = thiss.getElementsByTagName('item') |
|---|
| 357 | for d in dil: |
|---|
| 358 | thiss.removeChild(d) |
|---|
| 359 | for i in iDoc.getElementsByTagName('item'): |
|---|
| 360 | item = self.doc.createElement( 'item' ) |
|---|
| 361 | thisl = i.getAttribute( 'label' ) |
|---|
| 362 | for k in ['title','description','label']: |
|---|
| 363 | item.setAttribute( k, i.getAttribute( k ) ) |
|---|
| 364 | uid = 'TAG:%s' % thisl |
|---|
| 365 | item.setAttribute( 'uid', uid ) |
|---|
| 366 | thiss.appendChild( item ) |
|---|
| 367 | |
|---|
| 368 | def writeRelations(self): |
|---|
| 369 | thiss = self.doc.getElementsByTagName( 'varRelations' )[0] |
|---|
| 370 | dil = thiss.getElementsByTagName('item') |
|---|
| 371 | for d in dil: |
|---|
| 372 | thiss.removeChild(d) |
|---|
| 373 | for i in self.fix.relations: |
|---|
| 374 | item = self.doc.createElement( 'item' ) |
|---|
| 375 | for j in range( len( self.fix.relationinfo ) ): |
|---|
| 376 | item.setAttribute( self.fix.relationinfo[j], i[j] ) |
|---|
| 377 | thiss.appendChild( item ) |
|---|
| 378 | |
|---|
| 379 | thiss = self.doc.getElementsByTagName( 'varRelLnk' )[0] |
|---|
| 380 | dil = thiss.getElementsByTagName('item') |
|---|
| 381 | for d in dil: |
|---|
| 382 | thiss.removeChild(d) |
|---|
| 383 | cc = collections.defaultdict(int) |
|---|
| 384 | for i in self.fix.relationlinks: |
|---|
| 385 | item = self.doc.createElement( 'item' ) |
|---|
| 386 | tvp = i[1] |
|---|
| 387 | ix = cc[tvp] |
|---|
| 388 | cc[tvp] += 1 |
|---|
| 389 | for j in range( len( self.fix.relationlinkinfo ) ): |
|---|
| 390 | item.setAttribute( self.fix.relationlinkinfo[j], i[0][j] ) |
|---|
| 391 | |
|---|
| 392 | if tvp in self.cmvUidByTVP: |
|---|
| 393 | vid = self.cmvUidByTVP[ tvp ][ix] |
|---|
| 394 | item.setAttribute( 'rid', vid ) |
|---|
| 395 | print 'INFO.093.00011: found: %s' % str(tvp) |
|---|
| 396 | else: |
|---|
| 397 | t,v,p = tvp |
|---|
| 398 | if ('CMIP5_%s' % t,v,p) in self.cmvUidByTVP: |
|---|
| 399 | if ix >= len(self.cmvUidByTVP[ ('CMIP5_%s' % t,v,p) ] ): |
|---|
| 400 | print 'WARN.093.00011b: not found: %s, %s' % (str(tvp),ix) |
|---|
| 401 | else: |
|---|
| 402 | vid = self.cmvUidByTVP[ ('CMIP5_%s' % t,v,p) ][ix] |
|---|
| 403 | item.setAttribute( 'rid', vid ) |
|---|
| 404 | print 'INFO.093.00011a: found: %s' % str(tvp) |
|---|
| 405 | elif t[-2:] in ['pt','zm'] and ( t[:-2],v,p) in self.cmvUidByTVP: |
|---|
| 406 | vid = self.cmvUidByTVP[ ( t[:-2],v,p) ][ix] |
|---|
| 407 | item.setAttribute( 'rid', vid ) |
|---|
| 408 | print 'INFO.093.00011a: found: %s' % str(tvp) |
|---|
| 409 | else: |
|---|
| 410 | print 'WARN.093.00011: not found: %s' % str(tvp) |
|---|
| 411 | thiss.appendChild( item ) |
|---|
| 412 | |
|---|
| 413 | def writeCellMethods(self): |
|---|
| 414 | thiss = self.doc.getElementsByTagName( 'cellMethods' )[0] |
|---|
| 415 | dil = thiss.getElementsByTagName('item') |
|---|
| 416 | for d in dil: |
|---|
| 417 | thiss.removeChild(d) |
|---|
| 418 | ss = set() |
|---|
| 419 | for k in WbMods.cellm.cminfo.keys(): |
|---|
| 420 | t,l,u,d = WbMods.cellm.cminfo[k] |
|---|
| 421 | item = self.doc.createElement( 'item' ) |
|---|
| 422 | item.setAttribute( 'label', l ) |
|---|
| 423 | item.setAttribute( 'title', t ) |
|---|
| 424 | item.setAttribute( 'cell_methods', k ) |
|---|
| 425 | item.setAttribute( 'uid', u ) |
|---|
| 426 | if d != None: |
|---|
| 427 | item.setAttribute( 'description', d ) |
|---|
| 428 | thiss.appendChild( item ) |
|---|
| 429 | |
|---|
| 430 | def writeMipTables(self): |
|---|
| 431 | self.cfgUids = set() |
|---|
| 432 | xx = self.vocabs.getElementsByTagName( 'table' ) |
|---|
| 433 | assert len(xx) == 1, 'Expecting one element named "table", found %s' % len(xx) |
|---|
| 434 | iDoc = xx[0] |
|---|
| 435 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
|---|
| 436 | thiss = self.doc.getElementsByTagName( 'miptable' )[0] |
|---|
| 437 | dil = thiss.getElementsByTagName('item') |
|---|
| 438 | for d in dil: |
|---|
| 439 | thiss.removeChild(d) |
|---|
| 440 | ss = set() |
|---|
| 441 | for i in iDoc.getElementsByTagName('item'): |
|---|
| 442 | thisl = i.getAttribute( 'label' ) |
|---|
| 443 | thisll = string.replace( thisl, '_', '-' ) |
|---|
| 444 | if thisl in self.tables: |
|---|
| 445 | item = self.doc.createElement( 'item' ) |
|---|
| 446 | ss.add(thisl) |
|---|
| 447 | item.setAttribute( 'label', thisll ) |
|---|
| 448 | item.setAttribute( 'uid', 'MIPtable::%s' % thisl ) |
|---|
| 449 | for k in ['title','description','frequency']: |
|---|
| 450 | item.setAttribute( k, i.getAttribute( k ) ) |
|---|
| 451 | thiss.appendChild( item ) |
|---|
| 452 | else: |
|---|
| 453 | print 'WARN.mtid.002: ignoring mip table record for %s' % thisl |
|---|
| 454 | for lab in self.tables: |
|---|
| 455 | if lab not in ss: |
|---|
| 456 | thisll = string.replace( lab, '_', '-' ) |
|---|
| 457 | print 'WARN.mtid.001: creating dummy mip table record for %s' % lab |
|---|
| 458 | item = self.doc.createElement( 'item' ) |
|---|
| 459 | item.setAttribute( 'label', thisll ) |
|---|
| 460 | item.setAttribute( 'title', lab ) |
|---|
| 461 | item.setAttribute( 'frequency', '???' ) |
|---|
| 462 | item.setAttribute( 'uid', 'MIPtable::%s' % lab ) |
|---|
| 463 | thiss.appendChild( item ) |
|---|
| 464 | |
|---|
| 465 | ################################################################## |
|---|
| 466 | |
|---|
| 467 | def writeMcfg(self): |
|---|
| 468 | self.cfgUids = set() |
|---|
| 469 | xx = self.vocabs.getElementsByTagName( 'modelConfig' ) |
|---|
| 470 | assert len(xx) == 1, 'Expecting one element named "modelConfig", found %s' % len(xx) |
|---|
| 471 | iDoc = xx[0] |
|---|
| 472 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
|---|
| 473 | thiss = self.doc.getElementsByTagName( 'modelConfig' )[0] |
|---|
| 474 | dil = thiss.getElementsByTagName('item') |
|---|
| 475 | for d in dil: |
|---|
| 476 | thiss.removeChild(d) |
|---|
| 477 | for i in iDoc.getElementsByTagName('item'): |
|---|
| 478 | item = self.doc.createElement( 'item' ) |
|---|
| 479 | thisl = i.getAttribute( 'label' ) |
|---|
| 480 | for k in ['title','MIPs','usage','range','type']: |
|---|
| 481 | item.setAttribute( k, i.getAttribute( k ) ) |
|---|
| 482 | uid = str( thisl ) |
|---|
| 483 | item.setAttribute( 'uid', uid ) |
|---|
| 484 | item.setAttribute( 'label', uid ) |
|---|
| 485 | self.cfgUids.add( uid ) |
|---|
| 486 | thiss.appendChild( item ) |
|---|
| 487 | |
|---|
| 488 | def writeCmDim(self): |
|---|
| 489 | thiss = self.doc.getElementsByTagName( 'grids' )[0] |
|---|
| 490 | dil = thiss.getElementsByTagName('item') |
|---|
| 491 | for d in dil: |
|---|
| 492 | thiss.removeChild(d) |
|---|
| 493 | kk = ['tables','label','altLabel','description','standardName','title','axis','units','isIndex','coords','bounds','direction','valid_min','valid_max','type','positive','value','boundsValues','requested','boundsRequested','tolRequested','isGrid' ] |
|---|
| 494 | for j in self.cmDims.ss.keys(): |
|---|
| 495 | item = self.doc.createElement( 'item' ) |
|---|
| 496 | for i in range(len(kk)): |
|---|
| 497 | if kk[i] in ['valid_min','valid_max']: |
|---|
| 498 | if str( self.cmDims.ss[j][i] ) != '': |
|---|
| 499 | item.setAttribute( kk[i], str( self.cmDims.ss[j][i] ) ) |
|---|
| 500 | else: |
|---|
| 501 | item.setAttribute( kk[i], str( self.cmDims.ss[j][i] ) ) |
|---|
| 502 | uid = 'dim:%s' % str( self.cmDims.ss[j][1] ) |
|---|
| 503 | item.setAttribute( 'uid', uid ) |
|---|
| 504 | thiss.appendChild( item ) |
|---|
| 505 | |
|---|
| 506 | def writeVar(self): |
|---|
| 507 | thiss = self.doc.getElementsByTagName( 'var' )[0] |
|---|
| 508 | dil = thiss.getElementsByTagName('item') |
|---|
| 509 | for d in dil: |
|---|
| 510 | thiss.removeChild(d) |
|---|
| 511 | #<item description="dummyAt" id="001.001.001" label="example-01" procComment="dummyAt" procnote="dummyAt" prov="dummyAt" provmip="dummyAt" sn="dummyAt" title="dummy title string" uid="fa349c44-9ccb-11e5-8176-5404a60d96b5" units="dummyAt"/> |
|---|
| 512 | kk= ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp','replaced_by'] |
|---|
| 513 | self.usedSn = set() |
|---|
| 514 | for uid in self.importWbMods.vars.d1: |
|---|
| 515 | item = self.doc.createElement( 'item' ) |
|---|
| 516 | thisr = self.importWbMods.vars.d1[uid] |
|---|
| 517 | sn = thisr[7] |
|---|
| 518 | self.usedSn.add( sn ) |
|---|
| 519 | if sn.find( ' ' ) != -1: |
|---|
| 520 | print 'WARN.sn.00001: space in sn: ',sn |
|---|
| 521 | thisr[7] = sn.replace( ' ', '_' ) |
|---|
| 522 | if sn.find( '?' ) != -1: |
|---|
| 523 | print 'WARN.sn.00002: question in sn: ',sn |
|---|
| 524 | thisr[7] = sn.replace( '?', '' ) |
|---|
| 525 | |
|---|
| 526 | for i in range( len(kk) -2 ): |
|---|
| 527 | item.setAttribute( kk[i], thisr[i] ) |
|---|
| 528 | if len( self.importWbMods.vars.c1[ thisr[0] ] ) == 1: |
|---|
| 529 | id = thisr[0] |
|---|
| 530 | else: |
|---|
| 531 | id = '%s.%s' % (thisr[0],self.importWbMods.vars.c1[ thisr[0] ].index( uid ) ) |
|---|
| 532 | print 'INFO.090.00050: setting id for non-unique variable name: %s' % id |
|---|
| 533 | item.setAttribute( 'id', id ) |
|---|
| 534 | thiss.appendChild( item ) |
|---|
| 535 | |
|---|
| 536 | def writeRvg(self): |
|---|
| 537 | thiss = self.doc.getElementsByTagName( 'requestVarGroup' )[0] |
|---|
| 538 | dil = thiss.getElementsByTagName('item') |
|---|
| 539 | for d in dil: |
|---|
| 540 | thiss.removeChild(d) |
|---|
| 541 | #<item description="dummyAt" id="001.001.001" label="example-01" procComment="dummyAt" procnote="dummyAt" prov="dummyAt" provmip="dummyAt" sn="dummyAt" title="dummy title string" uid="fa349c44-9ccb-11e5-8176-5404a60d96b5" units="dummyAt"/> |
|---|
| 542 | kk= ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp','replaced_by'] |
|---|
| 543 | kk= ['uid', 'label', 'title', 'mip', 'ref', 'refNote'] |
|---|
| 544 | for uid in self.importWbMods.vg.vg: |
|---|
| 545 | item = self.doc.createElement( 'item' ) |
|---|
| 546 | thisr = self.importWbMods.vg.vg[uid] |
|---|
| 547 | for i in range( len(kk)): |
|---|
| 548 | item.setAttribute( kk[i], thisr[i] ) |
|---|
| 549 | ##item.setAttribute( 'id', id ) |
|---|
| 550 | thiss.appendChild( item ) |
|---|
| 551 | |
|---|
| 552 | def writeRvx(self): |
|---|
| 553 | thiss = self.doc.getElementsByTagName( 'requestVar' )[0] |
|---|
| 554 | |
|---|
| 555 | ## 'vid', 'title', 'label', 'priority', 'vgid', 'mip' |
|---|
| 556 | cols = self.rq.rqvarx['__cols__'] |
|---|
| 557 | nn = 0 |
|---|
| 558 | for k in self.rq.rqvarx.keys(): |
|---|
| 559 | if k[0] != '_': |
|---|
| 560 | rec = self.rq.rqvarx[k] |
|---|
| 561 | print 'INFO.writeRvx.01001: ',k,rec |
|---|
| 562 | item = self.doc.createElement( 'item' ) |
|---|
| 563 | item.setAttribute( 'uid', k ) |
|---|
| 564 | for i in range(6): |
|---|
| 565 | item.setAttribute( cols[i], rec[i] ) |
|---|
| 566 | vid = item.getAttribute( 'vid' ) |
|---|
| 567 | |
|---|
| 568 | if vid in self.cmvMap002: |
|---|
| 569 | vid = self.cmvMap002[vid] |
|---|
| 570 | item.setAttribute( 'vid', vid ) |
|---|
| 571 | if vid in extRefCmvMapped: |
|---|
| 572 | vid = extRefCmvMapped[vid] |
|---|
| 573 | item.setAttribute( 'vid', vid ) |
|---|
| 574 | |
|---|
| 575 | if vid not in self.cmvByUid: |
|---|
| 576 | print 'ERROR.fixcmv.0005: attempt to use invalid vid (writeRvx): ',vid,k,rec |
|---|
| 577 | thiss.appendChild( item ) |
|---|
| 578 | nn += 1 |
|---|
| 579 | print 'INFO.writeRvx.00001: %s records written' % nn |
|---|
| 580 | |
|---|
| 581 | def writeSn(self): |
|---|
| 582 | import loadcf |
|---|
| 583 | cf = loadcf.cf() |
|---|
| 584 | thiss = self.doc.getElementsByTagName( 'standardname' )[0] |
|---|
| 585 | dil = thiss.getElementsByTagName('item') |
|---|
| 586 | for d in dil: |
|---|
| 587 | thiss.removeChild(d) |
|---|
| 588 | |
|---|
| 589 | for i in sorted( cf.names.keys() ): |
|---|
| 590 | d,u = cf.names[i] |
|---|
| 591 | self.appendSnItem(thiss,i,d,u) |
|---|
| 592 | |
|---|
| 593 | for i in sorted( cf.alias.keys() ): |
|---|
| 594 | if i in self.usedSn: |
|---|
| 595 | print 'WARN.aliassn.00001: standard name alias: %s' % i |
|---|
| 596 | sn = cf.alias[i] |
|---|
| 597 | d,u = cf.names[sn] |
|---|
| 598 | self.appendSnItem(thiss,i,d,u) |
|---|
| 599 | |
|---|
| 600 | def appendSnItem(self,thiss,sn,desc,units): |
|---|
| 601 | item = self.doc.createElement( 'item' ) |
|---|
| 602 | uid = str(sn) |
|---|
| 603 | if uid.find( ' ' ) != -1: |
|---|
| 604 | print 'SEVERE.sn.00001: BAD SN: %s' % uid |
|---|
| 605 | uid = uid.replace( ' ', '' ) |
|---|
| 606 | title = string.capwords( string.replace( uid, '_', ' ' ) ) |
|---|
| 607 | label = string.replace( title, ' ', '' ) |
|---|
| 608 | for w in ['Of','In','On','By']: |
|---|
| 609 | ww = ' %s ' % w |
|---|
| 610 | title = title.replace( ww, string.lower( ww ) ) |
|---|
| 611 | item.setAttribute( 'uid', uid ) |
|---|
| 612 | item.setAttribute( 'label', label ) |
|---|
| 613 | item.setAttribute( 'title', title ) |
|---|
| 614 | item.setAttribute( 'units', str(units) ) |
|---|
| 615 | item.setAttribute( 'description', uniCleanFunc( desc ) ) |
|---|
| 616 | thiss.appendChild( item ) |
|---|
| 617 | |
|---|
| 618 | def writeTimeSlice(self): |
|---|
| 619 | xx = self.vocabs.getElementsByTagName( 'timeSlice' ) |
|---|
| 620 | assert len(xx) == 1, 'Expecting one element named "activity", found %s' % len(xx) |
|---|
| 621 | iDoc = xx[0] |
|---|
| 622 | thiss = self.doc.getElementsByTagName( 'timeSlice' )[0] |
|---|
| 623 | dil = thiss.getElementsByTagName('item') |
|---|
| 624 | for d in dil: |
|---|
| 625 | thiss.removeChild(d) |
|---|
| 626 | |
|---|
| 627 | for i in iDoc.getElementsByTagName('item'): |
|---|
| 628 | item = self.doc.createElement( 'item' ) |
|---|
| 629 | label = i.getAttribute( 'label' ) |
|---|
| 630 | for k in ['label','title','type']: |
|---|
| 631 | item.setAttribute( k, i.getAttribute( k ) ) |
|---|
| 632 | for k in ['child']: |
|---|
| 633 | if i.getAttribute( k ) != '': |
|---|
| 634 | item.setAttribute( k, i.getAttribute( k ) ) |
|---|
| 635 | for k in ['start', 'end', 'step', 'sliceLen', 'nyears']: |
|---|
| 636 | if i.getAttribute( k ) != '': |
|---|
| 637 | v = str( int( float( i.getAttribute( k ) ) ) ) |
|---|
| 638 | item.setAttribute( k, v ) |
|---|
| 639 | item.setAttribute( 'uid', '_slice_%s' % label ) |
|---|
| 640 | thiss.appendChild( item ) |
|---|
| 641 | |
|---|
| 642 | def writeMip(self): |
|---|
| 643 | self.mipUids = set() |
|---|
| 644 | xx = self.vocabs.getElementsByTagName( 'activity' ) |
|---|
| 645 | assert len(xx) == 1, 'Expecting one element named "activity", found %s' % len(xx) |
|---|
| 646 | mipDoc = xx[0] |
|---|
| 647 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
|---|
| 648 | thiss = self.doc.getElementsByTagName( 'mip' )[0] |
|---|
| 649 | dil = thiss.getElementsByTagName('item') |
|---|
| 650 | for d in dil: |
|---|
| 651 | thiss.removeChild(d) |
|---|
| 652 | for i in mipDoc.getElementsByTagName('item'): |
|---|
| 653 | item = self.doc.createElement( 'item' ) |
|---|
| 654 | mip = i.getAttribute( 'label' ) |
|---|
| 655 | for k in ['title','url']: |
|---|
| 656 | item.setAttribute( k, i.getAttribute( k ) ) |
|---|
| 657 | uid = string.replace( mip, ' ', '' ) |
|---|
| 658 | item.setAttribute( 'uid', uid ) |
|---|
| 659 | item.setAttribute( 'label', uid ) |
|---|
| 660 | self.mipUids.add( uid ) |
|---|
| 661 | thiss.appendChild( item ) |
|---|
| 662 | |
|---|
| 663 | if "DECK" not in self.mipUids: |
|---|
| 664 | item = self.doc.createElement( 'item' ) |
|---|
| 665 | item.setAttribute( 'uid', 'DECK' ) |
|---|
| 666 | item.setAttribute( 'url', '' ) |
|---|
| 667 | item.setAttribute( 'label', 'DECK' ) |
|---|
| 668 | item.setAttribute( 'title', 'DECK - set of standard CMIP runs' ) |
|---|
| 669 | self.mipUids.add( "DECK" ) |
|---|
| 670 | thiss.appendChild( item ) |
|---|
| 671 | |
|---|
| 672 | def prepVar(self): |
|---|
| 673 | thissh = self.skey['var'] |
|---|
| 674 | thatsh = self.skey['ovar'] |
|---|
| 675 | self.varsxx = None |
|---|
| 676 | self.varsxxx = None |
|---|
| 677 | dothis = False |
|---|
| 678 | if self.newImport: |
|---|
| 679 | #['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov', 'priority0', 'rlm0'] |
|---|
| 680 | self.vars = collections.defaultdict( list ) |
|---|
| 681 | for k in thissh: |
|---|
| 682 | if k[0] != '_': |
|---|
| 683 | ll = thissh[k][:] |
|---|
| 684 | thisl = str( ll[2] ) |
|---|
| 685 | self.vars[thisl].append( k ) |
|---|
| 686 | else: |
|---|
| 687 | if dothis: |
|---|
| 688 | self.varsxx = collections.defaultdict( list ) |
|---|
| 689 | for k in thissh: |
|---|
| 690 | if k[0] != '_': |
|---|
| 691 | ll = thissh[k][:] |
|---|
| 692 | self.varsxx[str( ll[0] )].append( k ) |
|---|
| 693 | else: |
|---|
| 694 | self.varsxxx = collections.defaultdict( list ) |
|---|
| 695 | for k in thatsh: |
|---|
| 696 | if k[0] != '_': |
|---|
| 697 | ll = thatsh[k][:] |
|---|
| 698 | thisl = str( ll[21] ) |
|---|
| 699 | self.varsxxx[thisl].append( k ) |
|---|
| 700 | |
|---|
| 701 | def prep(self,rq): |
|---|
| 702 | ## keys here match section element names in schema |
|---|
| 703 | #for k in [ 'var','ovar','groupitem', 'revisedtabitem']: |
|---|
| 704 | ## lists for 3 new sections |
|---|
| 705 | ### uuid, spid, tmid, cell_methods, cell_measures, description, procNote, prov |
|---|
| 706 | structure = [] |
|---|
| 707 | self.structInfo = {} |
|---|
| 708 | ### uuid, label, dimensions, shape, levels, levelFlag |
|---|
| 709 | spatialshp = [] |
|---|
| 710 | ### uuid, label, dimension, tstyle |
|---|
| 711 | temporalshp = [] |
|---|
| 712 | ### cmor variables |
|---|
| 713 | self.cmv = [] |
|---|
| 714 | self.rqvList = [] |
|---|
| 715 | self.rqvgList = [] |
|---|
| 716 | self.cmvUidByFL = collections.defaultdict( cls_s1 ) |
|---|
| 717 | self.cmvUidByTab = collections.defaultdict( cls_s1 ) |
|---|
| 718 | self.cmvUidByTVP = collections.defaultdict( list ) |
|---|
| 719 | self.cmvByLab = collections.defaultdict( dict ) |
|---|
| 720 | self.cmvs = set() |
|---|
| 721 | self.cmvinfo = ('uid','stid','label','title','description','vid', \ |
|---|
| 722 | 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', \ |
|---|
| 723 | 'deflate', 'valid_min', 'flag_values', 'modeling_realm', 'valid_max', 'positive', 'mipTable', \ |
|---|
| 724 | 'prov', 'provNote', 'frequency', 'rowIndex', 'coordinates','defaultPriority','mipTableSection','processing') |
|---|
| 725 | self.cmvinfo2 = ('uid','stid','label','title','description','vid', \ |
|---|
| 726 | 'deflate_level', 'shuffle', 'type', \ |
|---|
| 727 | 'deflate', 'modeling_realm', 'positive', 'mipTable', 'mipTableSection', \ |
|---|
| 728 | 'prov', 'provNote', 'frequency', 'rowIndex', 'defaultPriority','processing') |
|---|
| 729 | self.rqvinfo = ('uid','label','vid','gpid','priority','mip','table' ) |
|---|
| 730 | |
|---|
| 731 | |
|---|
| 732 | thissh = self.skey['var'] |
|---|
| 733 | self.varUidByName = self.importWbMods.vars.c1 |
|---|
| 734 | ##self.varUidByName = collections.defaultdict( list ) |
|---|
| 735 | if self.newImport: |
|---|
| 736 | oo = open('refDefaultP.txt', 'w' ) |
|---|
| 737 | for k in thissh: |
|---|
| 738 | if k[0] != '_': |
|---|
| 739 | ##['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov', 'priority0'] |
|---|
| 740 | thisuuid = str(k) |
|---|
| 741 | self.defaultP[thisuuid] = thissh[k][-1] |
|---|
| 742 | oo.write( '%s\t%s\t%s\t%s\n' % (k, thissh[k][0], thissh[k][2], thissh[k][-1] ) ) |
|---|
| 743 | ##self.varUidByName[thissh[k][0]].append( thisuuid ) |
|---|
| 744 | oo.close() |
|---|
| 745 | else: |
|---|
| 746 | ## ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp'] |
|---|
| 747 | self.vars = collections.defaultdict( list ) |
|---|
| 748 | self.vars1 = collections.defaultdict( list ) |
|---|
| 749 | for uid in self.importWbMods.vars.d1: |
|---|
| 750 | self.defaultP[uid] = self.importWbMods.vars.d1[uid][-1] |
|---|
| 751 | ## collection indexed by short_name |
|---|
| 752 | self.vars1[self.importWbMods.vars.d1[uid][0]].append( uid ) |
|---|
| 753 | ## collection indexed by standard_name |
|---|
| 754 | self.vars[self.importWbMods.vars.d1[uid][7]].append( uid ) |
|---|
| 755 | |
|---|
| 756 | ### request variables |
|---|
| 757 | self.rqv = [] |
|---|
| 758 | self.realmByVar = collections.defaultdict( list ) |
|---|
| 759 | for ksect in [ 'ovar','groupitem']: |
|---|
| 760 | thissh = self.skey[ksect] |
|---|
| 761 | if ksect == 'ovar': |
|---|
| 762 | setu1 = set() |
|---|
| 763 | kk = ['uid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid', 'processing', 'gpid','rowIndex','ssect', 'priority'] |
|---|
| 764 | id = kk.index('dimensions') |
|---|
| 765 | ivid = kk.index('vid') |
|---|
| 766 | thisnt = collections.namedtuple( 'ovar', kk ) |
|---|
| 767 | ov = [] |
|---|
| 768 | dset = set() |
|---|
| 769 | ds2 = set() |
|---|
| 770 | ovuu = collections.defaultdict( set ) |
|---|
| 771 | ovll = collections.defaultdict( list ) |
|---|
| 772 | for k in thissh: |
|---|
| 773 | if k[0] != '_': |
|---|
| 774 | rr = thissh[k][:] |
|---|
| 775 | rr[-3] = int( rr[-3] ) |
|---|
| 776 | if rr[id] == 's|c|a|l|a|r': |
|---|
| 777 | print 'ERROR.060.0001: bad dimension string detected and corrected: %s' % rr[id] |
|---|
| 778 | rr[id] = '' |
|---|
| 779 | if string.find( rr[23], 'plevs' ) != -1: |
|---|
| 780 | print 'INFO.qqqq01: ',rr[21:24] |
|---|
| 781 | rr[23] = string.replace(rr[23], 'plevs', 'plev19') |
|---|
| 782 | thisdim = rr[id] |
|---|
| 783 | if thisdim.find( ' ' ) != -1: |
|---|
| 784 | thisdim = '|'.join( [x.strip() for x in thisdim.split() ] ) |
|---|
| 785 | rr[id] = thisdim |
|---|
| 786 | |
|---|
| 787 | tt = thisnt._make( rr ) |
|---|
| 788 | if tt.uid in setu1: |
|---|
| 789 | print 'ERROR.099.0002: duplicate uuid: %s: %s' % (uuid,str(rr)) |
|---|
| 790 | if str(tt.vid) == '__var_not_found_012__': |
|---|
| 791 | print 'ERROR.099.00102: bad vid string: %s [%s]' % (tt.vid,tt.out_name) |
|---|
| 792 | |
|---|
| 793 | if tt.out_name == 'clisccp': |
|---|
| 794 | print 'INFO.clisccp.00002: ',rr,k |
|---|
| 795 | if tt.out_name == 'intppcalc': |
|---|
| 796 | print 'INFO.intppcalc.00002: ',rr,k |
|---|
| 797 | if tt.out_name == 'ch4': |
|---|
| 798 | print 'INFO.ch4.00002: ',rr,k |
|---|
| 799 | |
|---|
| 800 | if str(tt.vid) not in self.importWbMods.vars.d1: |
|---|
| 801 | if tt.out_name in self.importWbMods.vars.c1: |
|---|
| 802 | if len( self.importWbMods.vars.c1[ tt.out_name ] ) == 1: |
|---|
| 803 | rr[ivid] = self.importWbMods.vars.c1[ tt.out_name ][0] |
|---|
| 804 | tt = thisnt._make( rr ) |
|---|
| 805 | print 'INFO.099.00103: bad vid string replaced: %s [%s]' % (tt.vid,tt.out_name) |
|---|
| 806 | else: |
|---|
| 807 | print 'ERROR.099.00104: ambiguous bad vid string: %s [%s]' % (tt.vid,tt.out_name) |
|---|
| 808 | else: |
|---|
| 809 | print 'ERROR.099.00105: bad vid string: %s [%s]' % (tt.vid,tt.out_name) |
|---|
| 810 | setu1.add(tt.uid) |
|---|
| 811 | ov.append( tt ) |
|---|
| 812 | |
|---|
| 813 | dset.add( thisdim ) |
|---|
| 814 | if tt.out_name in ['al','albs']: |
|---|
| 815 | print 'INFO.albs.00001: ',thisdim, rr |
|---|
| 816 | if tt.dimensions == 'location|time1': |
|---|
| 817 | print 'INFO.dimensions.00001',rr |
|---|
| 818 | if tt.out_name == 'msftmyz': |
|---|
| 819 | print 'INFO.msftmyz.00002: ',self.cmv[-1] |
|---|
| 820 | |
|---|
| 821 | ds2.add( (tt.dimensions,tt.cell_methods,tt.cell_measures,tt.flag_values, tt.flag_meanings) ) |
|---|
| 822 | ovuu[tt.vid].add( tt.modeling_realm ) |
|---|
| 823 | ovll[tt.vid].append( tt ) |
|---|
| 824 | if rr[22][-3:] in ['mon','day','3hr']: |
|---|
| 825 | fr = rr[22][-3:] |
|---|
| 826 | elif rr[22][-2:] in [ 'yr','fx']: |
|---|
| 827 | fr = rr[22][-2:] |
|---|
| 828 | else: |
|---|
| 829 | fr = tab2freq[rr[22]] |
|---|
| 830 | |
|---|
| 831 | prov = '%s ((isd.003))' % tt.mipTable |
|---|
| 832 | if tt.mipTable[:4] == 'OMIP': |
|---|
| 833 | prov = 'CMIP6 [OMIP] ((isd.001))' |
|---|
| 834 | elif string.find( tt.mipTable, '_' ) != -1: |
|---|
| 835 | prov = '%s [%s] ((isd.002))' % tuple( string.split( tt.mipTable, '_' ) ) |
|---|
| 836 | |
|---|
| 837 | if tt.var[:5] == 'xx..ccmi_': |
|---|
| 838 | pass |
|---|
| 839 | assert False, 'blind pass disabled .....' |
|---|
| 840 | else: |
|---|
| 841 | lab = tt.var |
|---|
| 842 | if tt.out_name != tt.var: |
|---|
| 843 | print 'WARN.outname.00001: out name issue.... ',tt.out_name,tt.var |
|---|
| 844 | if lab in {'*','include Oyr 3D tracers'}: |
|---|
| 845 | lab = '-copy-' |
|---|
| 846 | print 'ERROR.copy.00090: ',tt |
|---|
| 847 | ## |
|---|
| 848 | ## insert conditional on redundant tables here ... |
|---|
| 849 | ## |
|---|
| 850 | if tt.mipTable in redundantTables: |
|---|
| 851 | if tt.mipTable not in reportedRedundantTables: |
|---|
| 852 | print 'INFO.000.11111: redundant table spotted: ',tt.mipTable |
|---|
| 853 | reportedRedundantTables.add( tt.mipTable ) |
|---|
| 854 | else: |
|---|
| 855 | if lab != '-copy-' and string.find( lab, '-' ) != -1: |
|---|
| 856 | print 'ERROR.207.00010: bad character in label: %s' % lab |
|---|
| 857 | lab = string.replace( lab, '-', '' ) |
|---|
| 858 | if tt.ssect != '': |
|---|
| 859 | ssect,sectid = tt.ssect |
|---|
| 860 | else: |
|---|
| 861 | ssect = '' |
|---|
| 862 | sectid = None |
|---|
| 863 | |
|---|
| 864 | uid = chkRefCmv( tt.mipTable, lab, tt.uid, 'cmv.01009' ) |
|---|
| 865 | if uid in self.cmvs: |
|---|
| 866 | print 'ERROR.090.09010: duplicate cmv uid %s : %s' % (uid, str(tt.__dict__) ) |
|---|
| 867 | self.cmvs.add( uid ) |
|---|
| 868 | self.cmv.append( (uid, (tt.dimensions,tt.cell_methods,tt.cell_measures,tt.flag_values,tt.flag_meanings), \ |
|---|
| 869 | lab, tt.title, tt.comment, tt.vid, \ |
|---|
| 870 | tt.deflate_level, tt.shuffle, tt.ok_max_mean_abs, tt.flag_meanings, tt.type, tt.ok_min_mean_abs, \ |
|---|
| 871 | tt.deflate, tt.valid_min, tt.flag_values, tt.modeling_realm, tt.valid_max, tt.positive, tt.mipTable, \ |
|---|
| 872 | prov, ssect, fr, tt.rowIndex, '',int(tt.priority), ssect, tt.processing) ) |
|---|
| 873 | if tt.ok_max_mean_abs != '': |
|---|
| 874 | try: |
|---|
| 875 | xx = float( tt.ok_max_mean_abs ) |
|---|
| 876 | except: |
|---|
| 877 | print 'FAILED to take float of ok_max_mean_abs -.....' |
|---|
| 878 | print tt.__dict__ |
|---|
| 879 | raise |
|---|
| 880 | if tt.out_name == 'intppcalc': |
|---|
| 881 | print 'INFO.intppcalc.00003: ',self.cmv[-1] |
|---|
| 882 | if tt.out_name == 'msftmyz': |
|---|
| 883 | print 'INFO.msftmyz.00003: ',self.cmv[-1] |
|---|
| 884 | if tt.processing != '': |
|---|
| 885 | print 'INFO.processing.0001: ',lab,tt.title,tt.processing |
|---|
| 886 | |
|---|
| 887 | self.realmByVar[ lab ].append( tt.modeling_realm ) |
|---|
| 888 | vgid = tt.gpid |
|---|
| 889 | self.rqv.append( (str( uuid.uuid1() ),lab, uid, vgid, tt.priority,'CMIP5',tt.mipTable) ) |
|---|
| 890 | if sectid not in (None, '_na_'): |
|---|
| 891 | self.rqv.append( (str( uuid.uuid1() ),lab, uid, sectid, tt.priority,'CMIP5',tt.mipTable) ) |
|---|
| 892 | if lab == 'somint': |
|---|
| 893 | print 'INFO.somint.0002:', self.rqv[-1] |
|---|
| 894 | if lab == 'sisnconc': |
|---|
| 895 | print 'INFO.sisnconc.0002:', self.rqv[-1] |
|---|
| 896 | thisd = {} |
|---|
| 897 | for k9 in range(len(self.rqvinfo)): |
|---|
| 898 | thisd[self.rqvinfo[k9]] = self.rqv[-1][k9] |
|---|
| 899 | self.rqvList.append( (thisd, 'prep:thisd' ) ) |
|---|
| 900 | |
|---|
| 901 | self.dsort( dset ) |
|---|
| 902 | print self.dsortdd[ksect][0].keys() |
|---|
| 903 | |
|---|
| 904 | elif ksect == 'groupitem': |
|---|
| 905 | kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'priority', 'mip', 'uid', 'oldv', 'rowIndex', 'cmorv', 'new', 'gpid', 'vkey', 'vid'] |
|---|
| 906 | thisnt = collections.namedtuple( 'groupitem', kk ) |
|---|
| 907 | il = kk.index( 'levels' ) |
|---|
| 908 | iu = kk.index( 'uid' ) |
|---|
| 909 | idx = kk.index( 'descriptionEx' ) |
|---|
| 910 | ixvid = kk.index( 'vid' ) |
|---|
| 911 | ixvar = kk.index( 'var' ) |
|---|
| 912 | ixcmv = kk.index( 'cmorv' ) |
|---|
| 913 | ixp = kk.index( 'priority' ) |
|---|
| 914 | dset = set() |
|---|
| 915 | dsgpi = set() |
|---|
| 916 | gi = [] |
|---|
| 917 | s1 = set() |
|---|
| 918 | sm = set() |
|---|
| 919 | ## |
|---|
| 920 | nov = 0 |
|---|
| 921 | setu = set() |
|---|
| 922 | for k in thissh.keys(): |
|---|
| 923 | if k[0] != '_': |
|---|
| 924 | rr = thissh[k][:] |
|---|
| 925 | if trace.GROUPITEM_ta and rr[ixvar] == 'ta': |
|---|
| 926 | print 'INFO.045.00004: groupitem: ',rr |
|---|
| 927 | |
|---|
| 928 | if rr[ixvar] == 'snc': |
|---|
| 929 | print 'INFO.snc.0010:',rr |
|---|
| 930 | ## |
|---|
| 931 | ## convert floats back to integer |
|---|
| 932 | ## |
|---|
| 933 | if type(rr[ixp]) == type(1.): |
|---|
| 934 | rr[ixp] = int(rr[ixp]) |
|---|
| 935 | ## |
|---|
| 936 | if rr[ixcmv] == '': |
|---|
| 937 | rr[ixcmv] = rr[1] |
|---|
| 938 | tt = thisnt._make( rr ) |
|---|
| 939 | |
|---|
| 940 | if tt.var == 'hus': |
|---|
| 941 | print 'INFO.hus.0001: ',tt.__dict__ |
|---|
| 942 | if str(tt.mip) != 'OMIP': |
|---|
| 943 | gi.append( tt ) |
|---|
| 944 | sm.add( tt.mask ) |
|---|
| 945 | |
|---|
| 946 | smdd = {} |
|---|
| 947 | dsn = {'floating ice shelf':u'floating ice sheet', u'ocean':u'sea', 'yes':'seaIce' } |
|---|
| 948 | d1 = {u'where land over land':(False,'area: mean where land over land'), \ |
|---|
| 949 | u'land':(False,'area: mean where land'), \ |
|---|
| 950 | u'hourly max':(False,'area: mean time: mean within hours time: maximum over hours'), \ |
|---|
| 951 | u'grounded ice sheet':(True,'area: mean where grounded_ice_sheet'), \ |
|---|
| 952 | u'ice sheet':(True,'area: mean where ice_sheet'), \ |
|---|
| 953 | u'floating ice sheet':(True,'area: mean where floating_ice_shelf'), \ |
|---|
| 954 | u'floating land ice':(True,'area: mean where floating_ice_shelf'), \ |
|---|
| 955 | u'where snow on sea ice':(True,'area: mean where snow over sea_ice'), \ |
|---|
| 956 | u'where melt ponds':(True,'area: mean where melt_pond over sea_ice'), \ |
|---|
| 957 | u'where ridge':(True,'area: mean where sea_ice_ridge over sea_ice'), \ |
|---|
| 958 | u'where sea ice':(True,'area: mean where sea_ice'), \ |
|---|
| 959 | u'floating ice shelf':(True,'area: mean where floating_ice_sheet'), \ |
|---|
| 960 | u'seaIce':(True,'area: mean where sea_ice over sea'), \ |
|---|
| 961 | u'where sea ice over sea':(True,'area: mean where sea_ice over sea'), \ |
|---|
| 962 | u'sea':(False,'area: mean where sea') } |
|---|
| 963 | |
|---|
| 964 | for s in sm: |
|---|
| 965 | if s in ['', 'no', u'none',u'Southern hemisphere', u'Northern hemisphere']: |
|---|
| 966 | smdd[s] = (False,'') |
|---|
| 967 | else: |
|---|
| 968 | s1 = string.strip(s) |
|---|
| 969 | try: |
|---|
| 970 | smdd[s] = d1[ dsn.get( s1,s1 ) ] |
|---|
| 971 | except: |
|---|
| 972 | print s, dsn.get( s1,s ) |
|---|
| 973 | raise |
|---|
| 974 | |
|---|
| 975 | eeu = dict() |
|---|
| 976 | |
|---|
| 977 | ccchk = ivg.checkCmv(gi) |
|---|
| 978 | nrep = 0 |
|---|
| 979 | |
|---|
| 980 | for tt in gi: |
|---|
| 981 | tabsect = '' |
|---|
| 982 | gr = 'a' |
|---|
| 983 | if tt.mip == 'OMIP' \ |
|---|
| 984 | or any( [string.find(tt.table,i) != -1 for i in ['Omon','Oyr','Oclim','OImon','Ocean'] ] ) \ |
|---|
| 985 | or any( [string.find(tt.group,i) != -1 for i in ['Omon','Oyr','Oclim','OImon','Ocean'] ] ) \ |
|---|
| 986 | or tt.cmorv in oceanVars: |
|---|
| 987 | gr = 'o' |
|---|
| 988 | elif tt.cmorv in siSurfFluxes: |
|---|
| 989 | gr = "M" |
|---|
| 990 | print 'INFO.simip.00010: ', tt.cmorv |
|---|
| 991 | |
|---|
| 992 | if self.varsxx != None: |
|---|
| 993 | if tt.cmorv in self.varsxx: |
|---|
| 994 | sss = set() |
|---|
| 995 | for k in self.varsxx[tt.cmorv]: |
|---|
| 996 | sss.add(self.skey['var'][k][-1] ) |
|---|
| 997 | sss = list(sss) |
|---|
| 998 | if len(sss) == 1: |
|---|
| 999 | if sss[0] == 'ocean': |
|---|
| 1000 | gr = 'o' |
|---|
| 1001 | elif sss[0] == 'atmos': |
|---|
| 1002 | gr = 'a' |
|---|
| 1003 | print 'INFO:lllw:', tt.cmorv, sss, gr |
|---|
| 1004 | elif self.varsxxx != None: |
|---|
| 1005 | if tt.cmorv in self.varsxxx: |
|---|
| 1006 | sss = set() |
|---|
| 1007 | for k in self.varsxxx[tt.cmorv]: |
|---|
| 1008 | sss.add(self.skey['ovar'][k][21] ) |
|---|
| 1009 | sss = list(sss) |
|---|
| 1010 | if len(sss) == 1: |
|---|
| 1011 | if sss[0] == 'ocean': |
|---|
| 1012 | gr = 'o' |
|---|
| 1013 | elif sss[0] == 'atmos': |
|---|
| 1014 | gr = 'a' |
|---|
| 1015 | print 'INFO:lllw:', tt.cmorv, sss, gr |
|---|
| 1016 | |
|---|
| 1017 | |
|---|
| 1018 | if tt.cmorv in ['talknat', 'sltbasin','clisccp','fCLandToOcean']: |
|---|
| 1019 | print 'INFO.llll: ',tt,gr |
|---|
| 1020 | ## |
|---|
| 1021 | ## adding label for mip here messes up referencing later ... |
|---|
| 1022 | ## |
|---|
| 1023 | if tt.mip in [ 'SIMIP','LUMIP']: |
|---|
| 1024 | grprov = tt.mip |
|---|
| 1025 | else: |
|---|
| 1026 | grprov = "" |
|---|
| 1027 | |
|---|
| 1028 | if tt.cmorv[-3:] in ['500','700','200','300','850','000']: |
|---|
| 1029 | print 'INFO.022.0002: Setting p... coordinate for ',tt |
|---|
| 1030 | if tt.cmorv[-5:] in [ 't2000']: |
|---|
| 1031 | thisc = 'depth2000m' |
|---|
| 1032 | elif tt.cmorv[-4:] in [ 't300', 't700']: |
|---|
| 1033 | thisc = 'depth%sm' % tt.cmorv[-3:] |
|---|
| 1034 | elif tt.cmorv[-3:] == '000': |
|---|
| 1035 | assert tt.cmorv[-4:] == '1000',str(tt) |
|---|
| 1036 | thisc = 'p%s' % tt.cmorv[-4:] |
|---|
| 1037 | else: |
|---|
| 1038 | thisc = 'p%s' % tt.cmorv[-3:] |
|---|
| 1039 | thissh = '%s|%s' % (tt.shape, thisc) |
|---|
| 1040 | else: |
|---|
| 1041 | thissh = tt.shape |
|---|
| 1042 | |
|---|
| 1043 | if thissh.find( ':' ) != -1: |
|---|
| 1044 | thissh, realmsuggest = thissh.split( ':' ) |
|---|
| 1045 | elif tt.cmorv in varRealmMapsInv: |
|---|
| 1046 | realmsuggest = varRealmMapsInv[tt.cmorv] |
|---|
| 1047 | else: |
|---|
| 1048 | realmsuggest = None |
|---|
| 1049 | |
|---|
| 1050 | ### need to check where variables are, perhaps, refering to "ovar" |
|---|
| 1051 | ##kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'misc', 'mip', 'uid', 'rowIndex', 'new', 'gpid', 'vkey', 'vid'] |
|---|
| 1052 | prov = '%s [%s]' % (tt.mip,tt.group) |
|---|
| 1053 | provNote = '' |
|---|
| 1054 | lab = tt.cmorv |
|---|
| 1055 | if lab in {'*','include Oyr 3D tracers'}: |
|---|
| 1056 | lab = '-copy-' |
|---|
| 1057 | elif lab == 'wap (omega)': |
|---|
| 1058 | lab = 'wap' |
|---|
| 1059 | provNote = 'wap (omega)' |
|---|
| 1060 | lab = string.replace( string.strip(lab), '_', '-' ) |
|---|
| 1061 | fr = freqmap.get( tt.freq, tt.freq ) |
|---|
| 1062 | assert tt.group not in ['icesheetmon', 'icesheetyear','icesheetfx'], 'Unexpected tt.group: %s' % str(tt) |
|---|
| 1063 | if tt.group in ['LImonant','LImongre','LIyrant','LIyrgre','LIfxant','LIfxgre']: |
|---|
| 1064 | gr = 'g' |
|---|
| 1065 | ntab = tt.group |
|---|
| 1066 | elif tt.group in ['ImonAnt','ImonGre','IyrAnt','IyrGre','IfxAnt','IfxGre']: |
|---|
| 1067 | gr = 'g' |
|---|
| 1068 | ntab = tt.group |
|---|
| 1069 | elif tt.group in ['icesheetmonant','icesheetmongre','icesheetfxant','icesheetfxgre']: |
|---|
| 1070 | gr = 'g' |
|---|
| 1071 | ntab = 'LI' + tt.group[8:] |
|---|
| 1072 | elif tt.group in ['icesheetyearant','icesheetyeargre']: |
|---|
| 1073 | gr = 'g' |
|---|
| 1074 | ntab = 'LIyr' + tt.group[-3:] |
|---|
| 1075 | elif tt.table in ['Omon','Oyr','day'] and tt.shape == '=': |
|---|
| 1076 | ntab = tt.table |
|---|
| 1077 | print 'INFO.table.02002: new request var for existing CMOR var: %s, %s' % (tt.table,lab) |
|---|
| 1078 | else: |
|---|
| 1079 | ntab = 'em%s' % string.capwords( fr ) |
|---|
| 1080 | |
|---|
| 1081 | if fr == '6hr' and tt.levels in ['7h',4.0, '4']: |
|---|
| 1082 | ntab = '6hrPlev' |
|---|
| 1083 | print 'INFO.033.00001: adding to 6hrPlev ....',lab, tt.__dict__ |
|---|
| 1084 | |
|---|
| 1085 | elif tt.gpid in self.importWbMods.vg.vg and self.importWbMods.vg.vg[tt.gpid][1] == 'ISMIP6-LImon-01': |
|---|
| 1086 | ##kk= ['uid', 'label', 'title', 'mip', 'ref', 'refNote'] |
|---|
| 1087 | |
|---|
| 1088 | ntab = 'LImon' |
|---|
| 1089 | tabsect = 'new' |
|---|
| 1090 | print 'INFO.limon.0001: Adding %s to LImon/new' % lab |
|---|
| 1091 | |
|---|
| 1092 | if thissh == 'YZT' and ntab[-1] != 'Z': |
|---|
| 1093 | ntab += 'Z' |
|---|
| 1094 | |
|---|
| 1095 | dset.add( (thissh, tt.levels,tt.tstyle,gr) ) |
|---|
| 1096 | if 'area: mean where ice_sheet' == smdd[tt.mask][1] and gr == 'o': |
|---|
| 1097 | print 'SEVERE.structure.30001: ',tt.__dict__,gr |
|---|
| 1098 | dsgpi.add( (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask], 'None', 'None', grprov) ) |
|---|
| 1099 | # |
|---|
| 1100 | # picking out re-purposed CMIP5 variables |
|---|
| 1101 | # |
|---|
| 1102 | if ovuu.has_key(tt.vid) and len( ovuu[tt.vid] ) == 1: |
|---|
| 1103 | print 'INFO.ovuu.00001: lookup started %s, %s' % (tt.cmorv,tt.group) |
|---|
| 1104 | if len( ovuu[tt.vid] ) > 1: |
|---|
| 1105 | print 'WARN.ovuu.00002: multiple realms: %s, %s: %s' % (tt.cmorv,tt.group, str( ovuu[tt.vid] ) ) |
|---|
| 1106 | tt1 = ovll[tt.vid][0] |
|---|
| 1107 | thisrealm = tt1.modeling_realm |
|---|
| 1108 | ttl = ovll[tt.vid][0].title |
|---|
| 1109 | p1 = ovll[tt.vid][0].priority |
|---|
| 1110 | #tt1 = ovuu[tt.vid] |
|---|
| 1111 | if thisrealm == 'seaIce': |
|---|
| 1112 | ntab = 'OImon' |
|---|
| 1113 | if tt.priority in [1,2,3]: |
|---|
| 1114 | p = tt.priority |
|---|
| 1115 | else: |
|---|
| 1116 | p = int(p1) |
|---|
| 1117 | |
|---|
| 1118 | if gr == 'o' and thisrealm == 'atmos': |
|---|
| 1119 | thisrealm = 'ocean' |
|---|
| 1120 | |
|---|
| 1121 | if string.find( tt.descriptionEx, '//' ) != -1: |
|---|
| 1122 | ttl += ' ' + string.split( tt.descriptionEx, '/' )[0] |
|---|
| 1123 | |
|---|
| 1124 | uid = chkRefCmv( ntab, lab, tt.uid, 'cmv.01001' ) |
|---|
| 1125 | if uid != tt.uid: |
|---|
| 1126 | self.cmvMap002[tt.uid] = uid |
|---|
| 1127 | |
|---|
| 1128 | thiscmv = (uid, (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask],tt1.flag_values,tt1.flag_meanings), |
|---|
| 1129 | lab, ttl, tt.descriptionEx, tt1.vid, \ |
|---|
| 1130 | tt1.deflate_level, tt1.shuffle, tt1.ok_max_mean_abs, tt1.flag_meanings, tt1.type, tt1.ok_min_mean_abs, \ |
|---|
| 1131 | tt1.deflate, tt1.valid_min, tt1.flag_values, thisrealm, tt1.valid_max, tt1.positive, ntab, \ |
|---|
| 1132 | prov, provNote, tt.freq, tt.rowIndex, '', int(p),tabsect, '') |
|---|
| 1133 | |
|---|
| 1134 | if type(tt1.valid_max) == type(1.) and tt1.valid_max <= tt1.valid_min: |
|---|
| 1135 | print 'ERROR.valmx.00001: valid_max below/equal valid_min: ',tt1 |
|---|
| 1136 | self.realmByVar[ lab ].append( thisrealm ) |
|---|
| 1137 | thisrqv = [str( uuid.uuid1() ),lab, uid, tt.gpid, p, tt.mip, tt.group] |
|---|
| 1138 | if lab in [ 'sisnconc','mrro','snc']: |
|---|
| 1139 | print 'INFO.%s.0001a:' % lab, thisrqv |
|---|
| 1140 | nov += 1 |
|---|
| 1141 | else: |
|---|
| 1142 | print 'INFO.ovuu.00003: lookup started %s, %s' % (tt.cmorv,tt.group) |
|---|
| 1143 | thisrealm = '' |
|---|
| 1144 | if realmsuggest != None: |
|---|
| 1145 | thisrealm = realmsuggest |
|---|
| 1146 | elif realmdefix.has_key( prov ): |
|---|
| 1147 | thisrealm = realmdefix[prov] |
|---|
| 1148 | else: |
|---|
| 1149 | print 'WARN:070.00001: realm not found for %s,%s' % (lab,prov) |
|---|
| 1150 | if gr == 'o' and thisrealm == 'atmos': |
|---|
| 1151 | thisrealm = 'ocean' |
|---|
| 1152 | |
|---|
| 1153 | if thisrealm == 'seaIce': |
|---|
| 1154 | if fr == 'mon': |
|---|
| 1155 | ntab = 'OImon' |
|---|
| 1156 | elif fr == 'day': |
|---|
| 1157 | ntab = 'SIday' |
|---|
| 1158 | else: |
|---|
| 1159 | raise |
|---|
| 1160 | |
|---|
| 1161 | if tt.priority in [1,2,3]: |
|---|
| 1162 | p = tt.priority |
|---|
| 1163 | ptr = 1 |
|---|
| 1164 | elif self.defaultP.get( tt.vid, 102) in [1,2,3]: |
|---|
| 1165 | p = self.defaultP.get( tt.vid, 102) |
|---|
| 1166 | ptr = 2 |
|---|
| 1167 | else: |
|---|
| 1168 | p = 1 |
|---|
| 1169 | print 'WARN:070.00002: priority defaulting to 1 for %s,%s' % (lab,prov) |
|---|
| 1170 | ptr = 3 |
|---|
| 1171 | |
|---|
| 1172 | if lab != '-copy-' and string.find( lab, '-' ) != -1: |
|---|
| 1173 | print 'ERROR.207.00020: bad character in label: %s' % lab |
|---|
| 1174 | lab = string.replace( lab, '-', '' ) |
|---|
| 1175 | ttl = '__from %s' % tt.table |
|---|
| 1176 | if lab != '-copy-': |
|---|
| 1177 | if tt.vid in self.importWbMods.vars.d1: |
|---|
| 1178 | ttl = self.importWbMods.vars.d1[tt.vid][1] |
|---|
| 1179 | if string.find( tt.descriptionEx, '//' ) != -1: |
|---|
| 1180 | ttl += ' ' + string.split( tt.descriptionEx, '/' )[0] |
|---|
| 1181 | |
|---|
| 1182 | if ntab in refCmv.refUid[ntab] and lab in refCmv.refUid[ntab]: |
|---|
| 1183 | uid = refCmv.refUid[ntab][lab] |
|---|
| 1184 | else: |
|---|
| 1185 | kk = str( '%s.%s' % (ntab,lab) ) |
|---|
| 1186 | if kk in extRefCmv: |
|---|
| 1187 | uid = extRefCmv[kk] |
|---|
| 1188 | print 'INFO.cmv.01003: uid found in extRedCmv: %s, %s (%s)' % (ntab,lab,uid) |
|---|
| 1189 | else: |
|---|
| 1190 | if tt.uid in extRefCmvInv: |
|---|
| 1191 | print 'SEVERE.cmv.09909: uid issue %s [%s] %s' % (kk,tt.uid,extRefCmvInv[tt.uid]) |
|---|
| 1192 | if extRefCmvMode != 'r': |
|---|
| 1193 | extRefCmv[kk] = tt.uid |
|---|
| 1194 | print 'ERROR.cmv.01003: uid not found: %s, %s (%s)' % (ntab,lab,tt.uid) |
|---|
| 1195 | uid = tt.uid |
|---|
| 1196 | |
|---|
| 1197 | if tt.levels in ['7h',]: |
|---|
| 1198 | if lab.find( tt.levels) == -1: |
|---|
| 1199 | print 'WARN.cmv.00880: extending label %s --> %s (%s)' % (lab,lab+tt.levels, str( tt.__dict__)) |
|---|
| 1200 | lab += tt.levels |
|---|
| 1201 | uid = chkRefCmv( ntab, lab, tt.uid, 'cmv.01011' ) |
|---|
| 1202 | if uid in eeu: |
|---|
| 1203 | print 'WARN.cmv.90001: duplicate cmv uid: ',uid,tt.__dict__,eeu[uid] |
|---|
| 1204 | thiscmv = (uid, (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask], 'None', 'None'), lab, ttl, tt.descriptionEx, tt.vid, \ |
|---|
| 1205 | 0, 0, None, 0, '', None, \ |
|---|
| 1206 | 0,None,0, thisrealm, None, '', ntab, \ |
|---|
| 1207 | prov, provNote, tt.freq, tt.rowIndex, '', str(p), tabsect, '') |
|---|
| 1208 | self.realmByVar[ lab ].append( thisrealm ) |
|---|
| 1209 | |
|---|
| 1210 | thisrqv = [str( uuid.uuid1() ),lab, uid, tt.gpid, p, tt.mip, tt.group] |
|---|
| 1211 | if lab in ['mrro', 'sisnconc','snc']: |
|---|
| 1212 | print 'INFO.%s.0001b:' % lab, thisrqv, ptr, thiscmv |
|---|
| 1213 | if trace.GROUPITEM_ta and lab == 'ta': |
|---|
| 1214 | print 'INFO.045.00005: groupitem: ',thisrqv |
|---|
| 1215 | |
|---|
| 1216 | if ccchk.uidReplace.has_key(thiscmv[0]): |
|---|
| 1217 | nrep += 1 |
|---|
| 1218 | print 'ERROR.cmv.09005: duplicate cmv uuid: %s: %s' % (uid, str(tt.__dict__) ) |
|---|
| 1219 | thisrqv[2] = ccchk.uidReplace[thiscmv[0]] |
|---|
| 1220 | else: |
|---|
| 1221 | if uid in setu1: |
|---|
| 1222 | print 'ERROR.cmv.09001: duplicate cmv uuid: %s: %s ' % (uid,str(tt.__dict__)) |
|---|
| 1223 | elif uid not in eeu: |
|---|
| 1224 | eeu[uid] = tt.__dict__ |
|---|
| 1225 | assert len(thiscmv) == len( self.cmvinfo ), 'BAD CMV specification length: %s' % str(cmv) |
|---|
| 1226 | self.cmv.append( thiscmv ) |
|---|
| 1227 | print 'INFO.cmv.x0001: append cmv: ',tt.__dict__ |
|---|
| 1228 | self.rqv.append( thisrqv ) |
|---|
| 1229 | if thisrqv[1] == 'somint': |
|---|
| 1230 | print 'INFO.somint.0003',thisrqv |
|---|
| 1231 | ## Variables marked with shape '?' from PMIP are actually defined by OMIP .... |
|---|
| 1232 | if tt.shape == '?': |
|---|
| 1233 | print 'ERROR.801.00001: shape not set: %s, %s, %s' % ( tt.shape,tt.mip,tt.var) |
|---|
| 1234 | |
|---|
| 1235 | ##['', u'none', u'land', u'grounded ice sheet', u'floating ice sheet', u'floating ice shelf', u'ocean', u'ice sheet', u'sea', u'Southern hemisphere', u'Northern hemisphere'] |
|---|
| 1236 | self.dsort( dset,mode=ksect ) |
|---|
| 1237 | print 'INFO.088.00001: nrep=%s' % nrep |
|---|
| 1238 | print 'INFO.088.00002: nov=%s [%s]' % (nov,len(gi)) |
|---|
| 1239 | |
|---|
| 1240 | nrep = [] |
|---|
| 1241 | for c in self.cmv: |
|---|
| 1242 | u,v,f,t,p,pn = (c[0],c[2],c[21],c[18],c[19],c[20]) |
|---|
| 1243 | self.cmvUidByFL[f].a[v].append( u ) |
|---|
| 1244 | self.cmvUidByTab[t].a[v].append( u ) |
|---|
| 1245 | if v[0] != '-': |
|---|
| 1246 | if (t,v,p) in self.cmvUidByTVP: |
|---|
| 1247 | nrep.append( (t,v,p) ) |
|---|
| 1248 | self.cmvUidByTVP[ (t,v,p) ].append( u ) |
|---|
| 1249 | self.cmvByLab[v][ u ] = (f,t) |
|---|
| 1250 | if len( nrep) > 0: |
|---|
| 1251 | print 'WARN.108.00088: unresolved .... %s' % len(nrep) |
|---|
| 1252 | for n in nrep: |
|---|
| 1253 | print n |
|---|
| 1254 | |
|---|
| 1255 | ####################################################################################### |
|---|
| 1256 | ################# |
|---|
| 1257 | std = set() |
|---|
| 1258 | ssd = set() |
|---|
| 1259 | esd = {} |
|---|
| 1260 | |
|---|
| 1261 | for ksect in [ 'ovar','groupitem']: |
|---|
| 1262 | #### create structure list for spatial and temporal dimensions |
|---|
| 1263 | for t in self.dsortdd[ksect][0].keys(): |
|---|
| 1264 | tdim, tt = self.dsort_u1( self.dsortdd[ksect][0][t], self.dsortdd[ksect][1] ) |
|---|
| 1265 | |
|---|
| 1266 | std.add( tdim ) |
|---|
| 1267 | ssd.add( tt ) |
|---|
| 1268 | |
|---|
| 1269 | etd = self.spsh.etd |
|---|
| 1270 | for s in std: |
|---|
| 1271 | assert s in etd, 'Temporal dimension record not found for %s (see importWbMods.loadSpatial)' % s |
|---|
| 1272 | |
|---|
| 1273 | kk = 0 |
|---|
| 1274 | self.spidInfo = {} |
|---|
| 1275 | for s in ssd: |
|---|
| 1276 | d,lfl,nl = s |
|---|
| 1277 | if d == 'longitude|latitude|plev7': |
|---|
| 1278 | print 'INFO.plev7.0002: resetting plev7 to plev7c' |
|---|
| 1279 | d = 'longitude|latitude|plev7c' |
|---|
| 1280 | if d == 'longitude|latitude|': |
|---|
| 1281 | print 'INFO.plev7.0003: removing trailing |' |
|---|
| 1282 | d = 'longitude|latitude' |
|---|
| 1283 | if d not in WbMods.ls.ss: |
|---|
| 1284 | print 'SEVERE.spatial.0001: dimensions not found in WbMods.ls: %s' % d |
|---|
| 1285 | assert False, 'Spatial shapes must be specified in spatialShape_ref.xls: %s' % d |
|---|
| 1286 | u = str( uuid.uuid1() ) |
|---|
| 1287 | lab = 'ssd-%3.3i' % kk |
|---|
| 1288 | description = 'ssd.%3.3i' % kk |
|---|
| 1289 | else: |
|---|
| 1290 | u = WbMods.ls.ss[d][4] |
|---|
| 1291 | lab = WbMods.ls.ss[d][0] |
|---|
| 1292 | description = WbMods.ls.ss[d][1] |
|---|
| 1293 | print 'INFO.spatial.0002: setting spatial shape uid: ',u,lab,description |
|---|
| 1294 | |
|---|
| 1295 | kk+=1 |
|---|
| 1296 | esd[s] = (u,lab,description,d,lfl,nl) |
|---|
| 1297 | self.spidInfo[u] = (u,lab,description,d,lfl,nl) |
|---|
| 1298 | |
|---|
| 1299 | ## clumsy here .. uid copied from spatialShape_ref.xls |
|---|
| 1300 | esd['Y-na'] = ('6dc25718-a24a-11e5-a29d-5404a60d96b5','extra','extra','latitude','false',1) |
|---|
| 1301 | self.dimsTime = etd |
|---|
| 1302 | self.dimsTimeInfo = ['uid','label','title','dimensions'] |
|---|
| 1303 | self.dimsSpace = esd |
|---|
| 1304 | self.dimsSpaceInfo = ['uid','label','title','dimensions','levelFlag','levels', 'dimids'] |
|---|
| 1305 | |
|---|
| 1306 | sshp = set() |
|---|
| 1307 | eshp = set() |
|---|
| 1308 | self.lls = ivg.structureList() |
|---|
| 1309 | nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tmid','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] ) |
|---|
| 1310 | kk = 0 |
|---|
| 1311 | strdd = {} |
|---|
| 1312 | ee3 = {} |
|---|
| 1313 | for dims,cmet,cmea,flgv, flgm in ds2: |
|---|
| 1314 | tdim, tt = self.dsort_u1( self.dsortdd['ovar'][0][dims], self.dsortdd['ovar'][1] ) |
|---|
| 1315 | d1,odim,d2,crd = self.dsortdd['ovar'][0][dims] |
|---|
| 1316 | sdim,lfl,nl = tt |
|---|
| 1317 | u = str( uuid.uuid1() ) |
|---|
| 1318 | spid = esd[tt][0] |
|---|
| 1319 | splab = esd[tt][1] |
|---|
| 1320 | tmid = etd[tdim][0] |
|---|
| 1321 | tlab = etd[tdim][1] |
|---|
| 1322 | kk += 1 |
|---|
| 1323 | label = 'str-a%3.3i' % kk |
|---|
| 1324 | proc = '' |
|---|
| 1325 | prov = 'CMIP5/OMIP' |
|---|
| 1326 | desc = '' |
|---|
| 1327 | ##print 'INFO.zzz01: ',dims, crd |
|---|
| 1328 | t0 = (dims,cmet,cmea,flgv,flgm) |
|---|
| 1329 | if cmet in ['None',None]: |
|---|
| 1330 | if tlab == 'time-mean': |
|---|
| 1331 | cmet = 'time: mean' |
|---|
| 1332 | elif tlab == 'time-point': |
|---|
| 1333 | cmet = 'time: point' |
|---|
| 1334 | else: |
|---|
| 1335 | cmet = '' |
|---|
| 1336 | if cmea in ['None',None]: |
|---|
| 1337 | cmea = '' |
|---|
| 1338 | |
|---|
| 1339 | if flgv in ['None',None,u'None']: |
|---|
| 1340 | flgv = '' |
|---|
| 1341 | if flgm in ['None',None,u'None']: |
|---|
| 1342 | flgm = '' |
|---|
| 1343 | t1 = (dims,cmet,cmea,flgv,flgm) |
|---|
| 1344 | ##errc,u,label = self.lls.add( [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] ) |
|---|
| 1345 | errc,u,label = self.lls.add2( [u,label,splab,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] ) |
|---|
| 1346 | print 'INFO.structure.00011:',t0,t1,u,label,spid,splab |
|---|
| 1347 | self.structInfo[u] = (label,cmet,spid,tmid) |
|---|
| 1348 | strdd[t0] = u |
|---|
| 1349 | strdd[t1] = u |
|---|
| 1350 | |
|---|
| 1351 | |
|---|
| 1352 | ttt = [ {u'Instantaneous (end of year)', 'instantaneous', 'point', 'Synoptic', u'synoptic', u'time: point'}, |
|---|
| 1353 | {'Mean', u'time: mean', u'Time mean', u'daily mean', u'mean', u'time mean', u'time: day', u'time: mean', u'weighted time mean', 'Cumulative annual fraction'}, set(), {''}, 'Daily max', 'Daily min'] |
|---|
| 1354 | cmetl = ['time: point', 'time: mean', 'time: mean within years time: mean over years ', '', 'time: maximum', 'time: minimum'] |
|---|
| 1355 | ##cmetlsi = ['time: point', ('time: mean area: mean where sea_ice over sea','time: mean where sea_ice area: mean where sea_ice'), 'time: mean within years time: mean over years ', ''] |
|---|
| 1356 | ee1 = {} |
|---|
| 1357 | for shp,lvls,sty,gr,tmsk, flgv, flgm, prov in dsgpi: |
|---|
| 1358 | t = (shp,lvls,sty,gr) |
|---|
| 1359 | ttx = (shp,lvls,sty,gr,tmsk, flgv, flgm, prov) |
|---|
| 1360 | if "t2000" in shp: |
|---|
| 1361 | print 'CCCC',shp,lvls,sty,gr,tmsk, flgv, flgm, prov |
|---|
| 1362 | tdim, tt = self.dsort_u1( self.dsortdd['groupitem'][0][t], self.dsortdd['groupitem'][1] ) |
|---|
| 1363 | d1,odim,d2,crd = self.dsortdd['groupitem'][0][t] |
|---|
| 1364 | tflm, msk = tmsk |
|---|
| 1365 | sdim,lfl,nl = tt |
|---|
| 1366 | u = str( uuid.uuid1() ) |
|---|
| 1367 | spid = esd[tt][0] |
|---|
| 1368 | splab = WbMods.ls.labByUid[ spid ] |
|---|
| 1369 | tmid = etd[tdim][0] |
|---|
| 1370 | tlab = etd[tdim][1] |
|---|
| 1371 | print 'INFO.spatial.6666: %s, %s, %s' % (splab,spid,str(tt)) |
|---|
| 1372 | |
|---|
| 1373 | if "XY" in splab: |
|---|
| 1374 | if gr == "M": |
|---|
| 1375 | cmea = "--MODEL" |
|---|
| 1376 | else: |
|---|
| 1377 | cmea = 'area: areacell%s' % gr |
|---|
| 1378 | else: |
|---|
| 1379 | cmea = '' |
|---|
| 1380 | |
|---|
| 1381 | cmet = '' |
|---|
| 1382 | for i in range(4): |
|---|
| 1383 | if sty in ttt[i]: |
|---|
| 1384 | if i == 1 and prov == 'SIMIP': |
|---|
| 1385 | if sty == u'weighted time mean': |
|---|
| 1386 | cmet = 'area: time: mean where sea_ice' |
|---|
| 1387 | elif sty == u'partial mean over sea': |
|---|
| 1388 | cmet = 'area: mean where sea_ice over sea time: mean' |
|---|
| 1389 | else: |
|---|
| 1390 | cmet = 'area: time: mean' |
|---|
| 1391 | elif prov == 'LUMIP' and shp == 'XYLT': |
|---|
| 1392 | print 'INFO.lumip.00505: ',shp,lvls,sty,gr,tmsk, flgv, flgm, prov |
|---|
| 1393 | if sty == 'Time mean': |
|---|
| 1394 | cmet = 'area: time: mean where landuse' |
|---|
| 1395 | elif sty == 'Cumulative annual fraction': |
|---|
| 1396 | cmet = 'area: mean time: sum' |
|---|
| 1397 | elif sty == 'Instantaneous (end of year)': |
|---|
| 1398 | cmet = 'area: mean where landUse time: point' |
|---|
| 1399 | else: |
|---|
| 1400 | cmet = cmetl[i] |
|---|
| 1401 | if cmet == '': |
|---|
| 1402 | cmet = msk |
|---|
| 1403 | elif msk != '': |
|---|
| 1404 | if "time:" in msk: |
|---|
| 1405 | print 'INFO.cmet.05005: using time: info in msk: ',msk,cmet |
|---|
| 1406 | cmet = msk |
|---|
| 1407 | else: |
|---|
| 1408 | cmet = msk + ' ' + cmet |
|---|
| 1409 | kk += 1 |
|---|
| 1410 | label = 'str-b%3.3i' % kk |
|---|
| 1411 | proc = '' |
|---|
| 1412 | if prov == '': |
|---|
| 1413 | prov = 'CMIP5, endorsed MIPs' |
|---|
| 1414 | desc = '' |
|---|
| 1415 | dims = tt[0] |
|---|
| 1416 | if tdim != '': |
|---|
| 1417 | dims += '|' + tdim |
|---|
| 1418 | if odim == 'spectband': |
|---|
| 1419 | print 'INFO.spectband.00002: ',tt, tdim, cmet, cmea, label, u |
|---|
| 1420 | |
|---|
| 1421 | if strdd.has_key( (dims,cmet,cmea,'','') ) and crd == '' and odim == '': |
|---|
| 1422 | u = strdd[ (dims,cmet,cmea,'','') ] |
|---|
| 1423 | ##print 'INFO.zzzy03: ',tt, tdim, cmet, cmea, label, u |
|---|
| 1424 | elif ee1.has_key( (tmid, spid, odim, cmet, cmea, crd) ): |
|---|
| 1425 | u = ee1[(tmid, spid, odim, cmet, cmea, crd)] |
|---|
| 1426 | ##print 'INFO.zzzy04: ', tt, tdim, cmet, label, u |
|---|
| 1427 | else: |
|---|
| 1428 | errc,u,label = self.lls.add2( [u,label,splab,tlab, odim, crd, cmet, cmea, '', '', desc, proc, prov] ) |
|---|
| 1429 | print 'INFO.zzzy02: ',tt, tdim, cmet, label, u, ttx |
|---|
| 1430 | |
|---|
| 1431 | ee1[(tmid, spid, odim, cmet, cmea, crd)] = u |
|---|
| 1432 | |
|---|
| 1433 | if sty == u'Synoptic': |
|---|
| 1434 | print 'yyyy: ',(shp,lvls,sty,gr,tmsk,flgv,flgm) |
|---|
| 1435 | strdd[(shp,lvls,sty,gr,tmsk,flgv,flgm)] = u |
|---|
| 1436 | if flgv == 'None' and flgm == 'None': |
|---|
| 1437 | strdd[(shp,lvls,sty,gr,tmsk,'','')] = u |
|---|
| 1438 | self.structInfo[u] = (label,cmet,spid,tmid) |
|---|
| 1439 | |
|---|
| 1440 | ##nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tmid','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] ) |
|---|
| 1441 | ##self.lls.close() |
|---|
| 1442 | self.strdd = strdd |
|---|
| 1443 | |
|---|
| 1444 | def cmvAudit(self,nn): |
|---|
| 1445 | nrep = [] |
|---|
| 1446 | self.cmvByUid = {} |
|---|
| 1447 | for c in self.cmv: |
|---|
| 1448 | u,v,f,t,p,pn = (c[0],c[2],c[21],c[18],c[19],c[20]) |
|---|
| 1449 | self.cmvUidByFL[f].a[v].append( u ) |
|---|
| 1450 | self.cmvUidByTab[t].a[v].append( u ) |
|---|
| 1451 | if v[0] != '-': |
|---|
| 1452 | if (t,v,p) in self.cmvUidByTVP: |
|---|
| 1453 | nrep.append( (t,v,p) ) |
|---|
| 1454 | self.cmvUidByTVP[ (t,v,p) ].append( u ) |
|---|
| 1455 | self.cmvByLab[v][ u ] = (f,t) |
|---|
| 1456 | self.cmvByUid[u] = c |
|---|
| 1457 | if len( nrep) > 0: |
|---|
| 1458 | print 'WARN.108.00088: unresolved .... %s' % len(nrep) |
|---|
| 1459 | for n in nrep: |
|---|
| 1460 | print n |
|---|
| 1461 | topop = [] |
|---|
| 1462 | cc1 = collections.defaultdict( int ) |
|---|
| 1463 | cc2 = collections.defaultdict( int ) |
|---|
| 1464 | for k in self.fix.fixcmv: |
|---|
| 1465 | vid = str( self.fix.fixcmv[k][4] ) |
|---|
| 1466 | fn = str( self.fix.fixcmv[k][6] ) |
|---|
| 1467 | cc1[fn] += 1 |
|---|
| 1468 | if vid not in self.cmvByUid: |
|---|
| 1469 | print 'WARN.fixcmv.0001: invalid vid %s [%s]' % ( vid, str( self.fix.fixcmv[k] ) ) |
|---|
| 1470 | topop.append( k ) |
|---|
| 1471 | cc2[fn] += 1 |
|---|
| 1472 | if len(topop) > 0: |
|---|
| 1473 | print 'WARN.fixcmv.0009: %s invalid vid value popped from fixcmv [%s]' % ( len( topop ), len( self.fix.fixcmv ) ) |
|---|
| 1474 | for k in topop: |
|---|
| 1475 | self.fix.fixcmv.pop( k ) |
|---|
| 1476 | for f in sorted( cc2.keys() ): |
|---|
| 1477 | print 'WARN.fixcmv.0010: %s invalid vid value popped from fixcmv:%s [%s]' % (cc2[f],f,cc1[f]) |
|---|
| 1478 | |
|---|
| 1479 | def dsort_u1(self,eed,sde): |
|---|
| 1480 | tdim = eed[2] |
|---|
| 1481 | sdim = eed[0] |
|---|
| 1482 | nl = sde[sdim] |
|---|
| 1483 | lfl = 'true' |
|---|
| 1484 | if nl == -1: |
|---|
| 1485 | lfl = 'false' |
|---|
| 1486 | nl = 0 |
|---|
| 1487 | elif nl == -2: |
|---|
| 1488 | nl = 1 |
|---|
| 1489 | pass |
|---|
| 1490 | return (tdim, (sdim,lfl,nl) ) |
|---|
| 1491 | |
|---|
| 1492 | |
|---|
| 1493 | def dsort( self, dset, mode='ovar' ): |
|---|
| 1494 | """Sort the dimensions string into spatial, temporal and other sections, |
|---|
| 1495 | returns a dictionary of tuples: spatial dimensions, other, temporal, coords and a 2nd, keyyed on spatial dimensions with a level count indicator. """ |
|---|
| 1496 | vdims = {'plev7':7, 'plev':4, 'plevs':19, 'plev3':3,'plev8':8,'alt40':40,'alev1':1, 'alevhalf':-1,'olevel':-1,'rho':-1,'sdepth':-1,'alevel':-1} |
|---|
| 1497 | odims = {'sza5','vegtype','scatratio','dbze','typepdec','vgidx','tau','effectRad'} |
|---|
| 1498 | dl = sorted( list(dset) ) |
|---|
| 1499 | if mode == 'ovar': |
|---|
| 1500 | eed = {} |
|---|
| 1501 | sss = [set(),set(),set(),set()] |
|---|
| 1502 | for s in dl: |
|---|
| 1503 | if s[:4] == 'time': |
|---|
| 1504 | bits = ['',s[4:]] |
|---|
| 1505 | else: |
|---|
| 1506 | bits = string.split(s, '|time', maxsplit=1 ) |
|---|
| 1507 | if len(bits) == 1: |
|---|
| 1508 | ts = '' |
|---|
| 1509 | else: |
|---|
| 1510 | ts = 'time' |
|---|
| 1511 | |
|---|
| 1512 | b0 = bits[0] |
|---|
| 1513 | b1 = '' |
|---|
| 1514 | bb = string.split(bits[0],'|') |
|---|
| 1515 | if bb[-1] in odims: |
|---|
| 1516 | if bb[-1] == 'tauxxx': |
|---|
| 1517 | b0 = bits[0][:-4] |
|---|
| 1518 | ts = 'tau|time' |
|---|
| 1519 | else: |
|---|
| 1520 | b0 = string.join( bb[:-1], '|' ) |
|---|
| 1521 | b1 = bb[-1] |
|---|
| 1522 | |
|---|
| 1523 | if len(bits) == 1: |
|---|
| 1524 | tup = (b0,b1,'','') |
|---|
| 1525 | elif len(bits) == 2: |
|---|
| 1526 | b2 = string.split( bits[1], '|', maxsplit=1 ) |
|---|
| 1527 | if len(b2) == 2: |
|---|
| 1528 | tup = (b0,b1, ts + b2[0], b2[1]) |
|---|
| 1529 | else: |
|---|
| 1530 | tup = (b0,b1, ts + bits[1], '') |
|---|
| 1531 | eed[s] = tup |
|---|
| 1532 | for k in range(4): |
|---|
| 1533 | sss[k].add(tup[k]) |
|---|
| 1534 | |
|---|
| 1535 | lcount = {} |
|---|
| 1536 | for s in sss[0]: |
|---|
| 1537 | bits = string.split( s, '|' ) |
|---|
| 1538 | nl = -2 |
|---|
| 1539 | thisb = None |
|---|
| 1540 | for b in bits: |
|---|
| 1541 | if vdims.has_key(b): |
|---|
| 1542 | assert nl == -2, 'Duplicate vertical level count in %s' % s |
|---|
| 1543 | nl = vdims[b] |
|---|
| 1544 | thisb = b |
|---|
| 1545 | lcount[s] = nl |
|---|
| 1546 | if nl == -1 and s == 'latitude|basin': |
|---|
| 1547 | print 'SEVERE: error in level deduction' |
|---|
| 1548 | print s, bits, nl, thisb |
|---|
| 1549 | raise |
|---|
| 1550 | else: |
|---|
| 1551 | ## input is a set of tuples, shape, levels, tstyle, o/a |
|---|
| 1552 | ## aim is to extract spatial, other, temporal dimensions + coords. |
|---|
| 1553 | sss = [set(),set(),set()] |
|---|
| 1554 | ttt = [ ('time1', u'Instantaneous (end of year)', 'instantaneous', 'point', 'Synoptic', u'synoptic', u'time: point'), |
|---|
| 1555 | ('Daily max', 'Daily min', 'time', 'Mean', u'time: mean', u'Time mean', u'daily mean', u'mean', u'time mean', u'time: day', u'time: mean', u'weighted time mean', 'Cumulative annual fraction','partial mean over sea'), |
|---|
| 1556 | (), |
|---|
| 1557 | ('diurnal','time3'), ('',)] |
|---|
| 1558 | ttdd = {} |
|---|
| 1559 | kkk = ['time1','time','time2','time3',''] |
|---|
| 1560 | for i in range(5): |
|---|
| 1561 | for k in ttt[i]: |
|---|
| 1562 | ttdd[k] = kkk[i] |
|---|
| 1563 | |
|---|
| 1564 | ##lll = {[u'1.0', u'14.0', u'16.0', u'17.0', u'27.0', u'3.0', u'36.0', u'4.0', u'40.0', u'7.0', ]:'flt', [u'17 (or 23 )', u'17 (or 23)', u'17 (or23)']:'obs', [u'Model levels or 27Plevs', u'all', u'all model levels', u'all model levels above 400hPa', u'all*', u'integrated over depth'], [u'single level',], [u'soil levels',], [u'surface',], [[u'100 m',] } |
|---|
| 1565 | dds = {'2D':'XYT', 'XY':'XYT', '':'?', '2D ':'XYT', 'XYZ':'XYZT', } |
|---|
| 1566 | ## 'K'?? 'XYK' |
|---|
| 1567 | ## |
|---|
| 1568 | ## |
|---|
| 1569 | ## spatial dimensions, other, time, coordinates, wild card in spatial dimensions. |
|---|
| 1570 | ## |
|---|
| 1571 | dd = {'XKT':('latitude|basin','','time','',False), \ |
|---|
| 1572 | 'XYZKT':('longitude|latitude|%s','tau','time','',True), \ |
|---|
| 1573 | 'XYZKmT':('longitude|latitude|%s','misrBands','time','',True), \ |
|---|
| 1574 | 'XYZKKT':('longitude|latitude|%s','effectRadXX|tau','time','',True), \ |
|---|
| 1575 | 'XYZKKlT':('longitude|latitude|%s','effectRadLi|tau','time','',True), \ |
|---|
| 1576 | 'XYZKKiT':('longitude|latitude|%s','effectRadIc|tau','time','',True), \ |
|---|
| 1577 | 'XYZKdbT':('longitude|latitude|%s','dbze','time','',True), \ |
|---|
| 1578 | 'XYZKscT':('longitude|latitude|%s','scatratio','time','',True), \ |
|---|
| 1579 | 'XYZT':('longitude|latitude|%s','','time','',True), \ |
|---|
| 1580 | 'XYST':('longitude|latitude|sdepth','','time','',False), \ |
|---|
| 1581 | 'XYSnT':('longitude|latitude|snowdepth','','time','',False), \ |
|---|
| 1582 | 'XYS1T':('longitude|latitude','sdepth1','time','',False), \ |
|---|
| 1583 | 'XYH1T':('longitude|latitude','height100','time','',False), \ |
|---|
| 1584 | 'XYZT*':('longitude|latitude|%s','','time','',True), \ |
|---|
| 1585 | 'KZT':('alevel','spectband','time','',False), \ |
|---|
| 1586 | 'Ksp':('','spectband','','',False), \ |
|---|
| 1587 | 'XYKspT':('longitude|latitude','spectband','time','',False), \ |
|---|
| 1588 | 'XYZKspT':('longitude|latitude|%s','spectband','time','',True), \ |
|---|
| 1589 | 'XYT':('longitude|latitude','','time','',False), \ |
|---|
| 1590 | '2DV':('longitude|latitude','','time','',False), \ |
|---|
| 1591 | 'XYKT':('longitude|latitude','sza5','time','',False), \ |
|---|
| 1592 | 'XYLT':('longitude|latitude','landUse','time','',False), \ |
|---|
| 1593 | 'XYant':('xant|yant','','time','',False), \ |
|---|
| 1594 | 'XYgre':('xgre|ygre','','time','',False), \ |
|---|
| 1595 | 'XYKszT':('longitude|latitude','sza5','time','',False), \ |
|---|
| 1596 | 'YZT':('latitude|%s','','time','',True), \ |
|---|
| 1597 | ## snowband was an error ... all the variables should have been on ice bands ... easier to correct here. |
|---|
| 1598 | 'XYKsn':('longitude|latitude','iceband','time','',False), \ |
|---|
| 1599 | 'XYKic':('longitude|latitude','iceband','time','',False), \ |
|---|
| 1600 | 'K4':('siline','','time','',False), \ |
|---|
| 1601 | 'scalar':('','','time','',False), \ |
|---|
| 1602 | '':('','','time','',False), \ |
|---|
| 1603 | 'sheet':('icesheet','','time','',False), \ |
|---|
| 1604 | 'ZST1':('alevel','site','time','',False), \ |
|---|
| 1605 | 'BasinYT':('latitude|basin','','time','',False), \ |
|---|
| 1606 | '?':( '?', '?','?','?',False) |
|---|
| 1607 | } |
|---|
| 1608 | |
|---|
| 1609 | for d in dl: |
|---|
| 1610 | for k in range(3): |
|---|
| 1611 | sss[k].add( d[k] ) |
|---|
| 1612 | |
|---|
| 1613 | ls0 = set() |
|---|
| 1614 | ls1 = set() |
|---|
| 1615 | ## |
|---|
| 1616 | ## sort shapes |
|---|
| 1617 | ## |
|---|
| 1618 | for s0 in sss[0]: |
|---|
| 1619 | if string.find( s0, '|' ) != -1: |
|---|
| 1620 | s,crd0 = string.split( s0, '|' ) |
|---|
| 1621 | else: |
|---|
| 1622 | s,crd0 = s0,None |
|---|
| 1623 | |
|---|
| 1624 | if not dd.has_key( dds.get(s,s) ): |
|---|
| 1625 | print 'ERROR.shape.0001: NO SOLUTION FOUND FOR SHAPE: "%s"' % s |
|---|
| 1626 | else: |
|---|
| 1627 | tt = dd[dds.get(s,s)] |
|---|
| 1628 | print 'INFO.structure.00077: mapping %s to %s' % (s,str(tt)) |
|---|
| 1629 | if tt[-1]: |
|---|
| 1630 | ls0.add( s0 ) |
|---|
| 1631 | ## |
|---|
| 1632 | ## look at shapes which require additional information about vertical coord |
|---|
| 1633 | ## |
|---|
| 1634 | for d in dl: |
|---|
| 1635 | if d[0] in ls0: |
|---|
| 1636 | ls1.add( d[1] ) |
|---|
| 1637 | |
|---|
| 1638 | lset = {'soil levels':'sdepth', 'all model levels above 400hPa':'alevel'} |
|---|
| 1639 | lsdd = {} |
|---|
| 1640 | print 'ls1....',ls1 |
|---|
| 1641 | for l in ls1: |
|---|
| 1642 | ii = 0 |
|---|
| 1643 | if l in ['all','all*','all model levels']: |
|---|
| 1644 | ii = -1 |
|---|
| 1645 | this = '%slevel' |
|---|
| 1646 | elif lset.has_key(l): |
|---|
| 1647 | ii = -1 |
|---|
| 1648 | this = lset[l] |
|---|
| 1649 | elif l == '': |
|---|
| 1650 | ii = -2 |
|---|
| 1651 | this = l |
|---|
| 1652 | elif l == '7h': |
|---|
| 1653 | ii = 7 |
|---|
| 1654 | this = 'plev7h' |
|---|
| 1655 | elif l == '3h': |
|---|
| 1656 | ii = 3 |
|---|
| 1657 | this = 'plev3h' |
|---|
| 1658 | else: |
|---|
| 1659 | ii = int( float(l) ) |
|---|
| 1660 | if ii in [40,16]: |
|---|
| 1661 | this = 'alt%s' % ii |
|---|
| 1662 | elif ii > 1.5: |
|---|
| 1663 | if ii > 35: |
|---|
| 1664 | ii = 39 |
|---|
| 1665 | ## |
|---|
| 1666 | ## merge 14 and 17 level specifications to a 19 level coordinate |
|---|
| 1667 | ## |
|---|
| 1668 | if ii in [14,17]: |
|---|
| 1669 | ii = 19 |
|---|
| 1670 | |
|---|
| 1671 | this = 'plev%s' % ii |
|---|
| 1672 | if ii == 7: |
|---|
| 1673 | this += 'c' |
|---|
| 1674 | |
|---|
| 1675 | else: |
|---|
| 1676 | this = '' |
|---|
| 1677 | lsdd[l] = (ii,this) |
|---|
| 1678 | ##['', 1.0, u'all', 3.0, 4.0, u'soil levels', 7.0, 40.0, u'all model levels', u'17 (or 23)', u'all*', 14.0, u'Model levels or 27Plevs', 16.0, 17.0, u'all model levels above 400hPa', u'17 (or23)', 36.0, u'17 (or 23 )', 27.0] |
|---|
| 1679 | |
|---|
| 1680 | lcount = {} |
|---|
| 1681 | eed = {} |
|---|
| 1682 | for d in dl: |
|---|
| 1683 | s0 = d[0] |
|---|
| 1684 | if string.find( s0, '|' ) != -1: |
|---|
| 1685 | s,crd0 = string.split( s0, '|' ) |
|---|
| 1686 | else: |
|---|
| 1687 | s,crd0 = s0,None |
|---|
| 1688 | if not dd.has_key( dds.get(s,s) ): |
|---|
| 1689 | print 'NO SOLUTION FOUND FOR SHAPE: "%s"' % s |
|---|
| 1690 | else: |
|---|
| 1691 | tt = dd[dds.get(s,s)] |
|---|
| 1692 | if tt[-1]: |
|---|
| 1693 | n,zz = lsdd[d[1]] |
|---|
| 1694 | if len(zz) > 0 and zz[0] == '%': |
|---|
| 1695 | zz = zz % d[3] |
|---|
| 1696 | sc = tt[0] % zz |
|---|
| 1697 | if sc[-1] == "|": |
|---|
| 1698 | print 'WARN.080.05000: Bad dimension found %s [%s] %s %s' % (sc,str(tt), str((n,zz)), str(d) ) |
|---|
| 1699 | else: |
|---|
| 1700 | sc = tt[0] |
|---|
| 1701 | if (string.find( sc, 'alevel' ) != -1) or (string.find( sc, 'olevel' ) != -1): |
|---|
| 1702 | n = -1 |
|---|
| 1703 | else: |
|---|
| 1704 | n = -2 |
|---|
| 1705 | |
|---|
| 1706 | lcount[sc] = n |
|---|
| 1707 | if n == -1 and sc == 'latitude|basin': |
|---|
| 1708 | print 'SEVERE [2]: error in level deduction' |
|---|
| 1709 | print sc, tt[0], tt[-1], n |
|---|
| 1710 | raise |
|---|
| 1711 | if crd0 != None: |
|---|
| 1712 | tt3 = tt[3] + crd0 |
|---|
| 1713 | else: |
|---|
| 1714 | tt3 = tt[3] |
|---|
| 1715 | |
|---|
| 1716 | if d[2] == '' and tt[2] not in ['','?']: |
|---|
| 1717 | d2 = tt[2] |
|---|
| 1718 | else: |
|---|
| 1719 | d2 = d[2] |
|---|
| 1720 | print 'INFO:qqq05: ',d, sc, tt, ttdd.get( d2, '**%s**' % d2), tt3 |
|---|
| 1721 | if d2 not in ttdd: |
|---|
| 1722 | print 'ERROR.time.00800: ',d2,ttdd.keys() |
|---|
| 1723 | eed[d] = (sc,tt[1],ttdd.get( d2, '**%s**' % d2),tt3) |
|---|
| 1724 | print 'INFO.ddddd.00002:',d,eed[d] |
|---|
| 1725 | self.dsortdd[mode] = (eed,lcount) |
|---|
| 1726 | |
|---|
| 1727 | def run(self,rq): |
|---|
| 1728 | |
|---|
| 1729 | xx = self.doc.getElementsByTagName( 'main' ) |
|---|
| 1730 | assert len(xx) == 1, 'Expecting one element named "main", found %s' % len(xx) |
|---|
| 1731 | main = xx[0] |
|---|
| 1732 | ## main = self.doc.childNodes[0] |
|---|
| 1733 | xsn = [] |
|---|
| 1734 | ff = {} |
|---|
| 1735 | for c in main.childNodes: |
|---|
| 1736 | tag = c.nodeName |
|---|
| 1737 | if tag != '#text': |
|---|
| 1738 | xsn.append(tag) |
|---|
| 1739 | il = [] |
|---|
| 1740 | for i in c.childNodes: |
|---|
| 1741 | if i.nodeName == 'item': |
|---|
| 1742 | il.append(i) |
|---|
| 1743 | xx = dict( il[0].attributes.items() ) |
|---|
| 1744 | ff[string.lower(tag)] = (c,xx) |
|---|
| 1745 | |
|---|
| 1746 | print xsn |
|---|
| 1747 | print ff.keys() |
|---|
| 1748 | self.ff = ff |
|---|
| 1749 | |
|---|
| 1750 | ## keys here match section element names in schema |
|---|
| 1751 | ## dec 2015 -- removing 'var' from this list |
|---|
| 1752 | assert self.schemaMode == 'dreq2', 'Old schema mode not supported here' |
|---|
| 1753 | sectlist = [ 'experiment','exptgroup','objective','requestlink', 'requestitem','requestvargroup','tablesection','requestvar' ] |
|---|
| 1754 | sectlist = [ 'experiment','exptgroup','objective','requestlink', 'requestitem','tablesection','requestvar' ] |
|---|
| 1755 | |
|---|
| 1756 | for k in sectlist: |
|---|
| 1757 | thissh = self.skey[k] |
|---|
| 1758 | ## set labmod False for section in which label coercion is not wanted |
|---|
| 1759 | labmod = True |
|---|
| 1760 | lll = [] |
|---|
| 1761 | for i in thissh.keys(): |
|---|
| 1762 | if i[0] != '_': |
|---|
| 1763 | lll.append(i) |
|---|
| 1764 | idk = 0 |
|---|
| 1765 | if len(lll) > 0: |
|---|
| 1766 | ## |
|---|
| 1767 | ## remove example items, xcept where this has been done in previous function |
|---|
| 1768 | ## |
|---|
| 1769 | if k not in ['xxxx']: |
|---|
| 1770 | dil = ff[k][0].getElementsByTagName('item') |
|---|
| 1771 | for d in dil: |
|---|
| 1772 | ff[k][0].removeChild(d) |
|---|
| 1773 | |
|---|
| 1774 | vare = {} |
|---|
| 1775 | nfr = set() |
|---|
| 1776 | for i in lll: |
|---|
| 1777 | ll = map( uniCleanFunc, thissh[i] ) |
|---|
| 1778 | assert len(ll) == len( thissh[i] ), 'Lost list elements after uniClean' |
|---|
| 1779 | idk += 1 |
|---|
| 1780 | ################################################################# |
|---|
| 1781 | ######### NEED TO CLEAN THIS IF ITEMS FROM OTHER SECTIONS ARE FILTERED |
|---|
| 1782 | ####################################################################### |
|---|
| 1783 | dothis = True |
|---|
| 1784 | if k not in ['var','experiment']: |
|---|
| 1785 | item = self.doc.createElement( 'item' ) |
|---|
| 1786 | |
|---|
| 1787 | if k == 'objective': |
|---|
| 1788 | ## - mip; tab; expt; rlid; ny |
|---|
| 1789 | labmod = False |
|---|
| 1790 | kk = ['mip', 'label', 'title', 'description'] |
|---|
| 1791 | ##thisl = string.replace(string.strip(str( ll[1] ) ), '-', ' ') |
|---|
| 1792 | ## camelcase this label. |
|---|
| 1793 | ##thisl = string.replace( string.capwords( thisl ), ' ', '' ) |
|---|
| 1794 | thisl = mycc( ll[1] ) |
|---|
| 1795 | mip = str( ll[0] ) |
|---|
| 1796 | self.objectives[mip][thisl] = i |
|---|
| 1797 | for j in range(len(kk)): |
|---|
| 1798 | thisv = str( ll[j] ) |
|---|
| 1799 | item.setAttribute( kk[j], thisv ) |
|---|
| 1800 | item.setAttribute( 'uid', str(i) ) |
|---|
| 1801 | elif k == 'experiment': |
|---|
| 1802 | labmod = False |
|---|
| 1803 | kk = ['uid','egid','label','description','mip','mcfg','tier','nstart','starty','endy','yps','ensz','ntot','comment'] |
|---|
| 1804 | thisl = str( ll[2] ) |
|---|
| 1805 | ## self.mips.add( str( ll[4] ) ) |
|---|
| 1806 | if ll[4] in ['SolarMIP']: |
|---|
| 1807 | dothis = False |
|---|
| 1808 | else: |
|---|
| 1809 | item = self.doc.createElement( 'item' ) |
|---|
| 1810 | for j in range(len(kk)): |
|---|
| 1811 | if kk[j] in ['tier','ensz']: |
|---|
| 1812 | if type(ll[j]) == type( 1 ): |
|---|
| 1813 | thisv = str( ll[j] ) |
|---|
| 1814 | else: |
|---|
| 1815 | thisv = string.join( [str(x) for x in ll[j]] ) |
|---|
| 1816 | else: |
|---|
| 1817 | thisv = str( uniCleanFunc(ll[j]) ) |
|---|
| 1818 | item.setAttribute( kk[j], thisv ) |
|---|
| 1819 | if kk[j] == 'yps': |
|---|
| 1820 | if type(ll[j]) == type([]): |
|---|
| 1821 | print 'ERROR.yps.00001: ', ll |
|---|
| 1822 | item.setAttribute( 'yps', '25' ) |
|---|
| 1823 | ################################################################################################## |
|---|
| 1824 | elif k == 'exptgroup': |
|---|
| 1825 | labmod = False |
|---|
| 1826 | kk = ['uid','label','tierMin','ntot'] |
|---|
| 1827 | thisl = str( ll[1] ) |
|---|
| 1828 | for j in range(len(kk)): |
|---|
| 1829 | thisv = str( ll[j] ) |
|---|
| 1830 | item.setAttribute( kk[j], thisv ) |
|---|
| 1831 | |
|---|
| 1832 | ################################################################################################## |
|---|
| 1833 | elif k == 'groupitem': |
|---|
| 1834 | kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'misc', 'mip', 'uid', 'new', 'gpid', 'vkey', 'vid'] |
|---|
| 1835 | assert len(kk) == len(ll), 'length mismatch, %s %s' % (len(kk),len(ll)) |
|---|
| 1836 | # - group; table; freq; descriptionEx; shape; levels; tstyle; mask; mip; mip2; uuid; new; vid |
|---|
| 1837 | ix_gpid = kk.index( 'gpid' ) |
|---|
| 1838 | thisl = string.strip(str( ll[1] ) ) |
|---|
| 1839 | for j in range(len(kk)): |
|---|
| 1840 | if j != 1: |
|---|
| 1841 | ##item.setAttribute( kk[j], str( ll[j] ) ) |
|---|
| 1842 | if kk[j] == 'misc': |
|---|
| 1843 | item.setAttribute( 'priority', str( ll[j] ) ) |
|---|
| 1844 | else: |
|---|
| 1845 | item.setAttribute( kk[j], str( ll[j] ) ) |
|---|
| 1846 | item.setAttribute( 'title', thisl ) |
|---|
| 1847 | if string.find(thisl, ' ') != -1: |
|---|
| 1848 | thisl = string.split(thisl)[0] |
|---|
| 1849 | thisuuid = ll[11] |
|---|
| 1850 | |
|---|
| 1851 | gpid = str(ll[ix_gpid]) |
|---|
| 1852 | if not self.importWbMods.vg.vg.has_key(gpid): |
|---|
| 1853 | print 'ERROR.015.0010: gpid not found %s' % str(ll) |
|---|
| 1854 | self.e15_10 += 1 |
|---|
| 1855 | if self.repl.has_key( thisuuid ): |
|---|
| 1856 | for tag,old,new in self.repl[thisuuid]: |
|---|
| 1857 | thisold = item.getAttribute( tag ) |
|---|
| 1858 | assert thisold == old, 'Attempt to replace value which is not present' |
|---|
| 1859 | item.setAttribute( tag, new ) |
|---|
| 1860 | |
|---|
| 1861 | #################################### from revised CMIP5 tables ### request variables referencing core tables |
|---|
| 1862 | elif k in ['requestvar']: |
|---|
| 1863 | #### need to fill gaps in variable groups --- or do better job upstream ########## !!!!!!!!!!!!!!!! |
|---|
| 1864 | ## - table; mip; uuid; priority |
|---|
| 1865 | ## |
|---|
| 1866 | ## variable group ids pulled through via mapping above .. very messy |
|---|
| 1867 | ## still missing some OMIP matches .... OMIP revised tables are not explicitly requested anywhere. |
|---|
| 1868 | ## |
|---|
| 1869 | kk = ['var', 'table', 'mip', 'vid', 'priority','vgid'] |
|---|
| 1870 | ##vgmaps = {'Oyr':'OMIP-Oyr','OImon':'SIMIP-seaicemon'} |
|---|
| 1871 | |
|---|
| 1872 | vgid = str( ll[5] ) |
|---|
| 1873 | doThis = True |
|---|
| 1874 | if vgid == '__unset__01__': |
|---|
| 1875 | dothis = False |
|---|
| 1876 | else: |
|---|
| 1877 | thisl = string.strip(str( ll[0] ) ) |
|---|
| 1878 | if len( thisl ) > 6 and thisl[-6:] == 'Adjust': |
|---|
| 1879 | dothis = False |
|---|
| 1880 | |
|---|
| 1881 | if dothis: |
|---|
| 1882 | thisuuid = str( uuid.uuid1() ) |
|---|
| 1883 | thisd = {} |
|---|
| 1884 | |
|---|
| 1885 | if ll[2] not in refMips: |
|---|
| 1886 | print 'SEVERE.mip.000001: bad mip label in request item: %s --- %s' % (ll[2],str(ll)) |
|---|
| 1887 | |
|---|
| 1888 | for j in range(len(kk)): |
|---|
| 1889 | if j != 0 and kk[j] != 'table': |
|---|
| 1890 | item.setAttribute( kk[j], str( ll[j] ) ) |
|---|
| 1891 | thisd[kk[j]] = str( ll[j] ) |
|---|
| 1892 | vid = str( ll[3] ) |
|---|
| 1893 | if vid in extRefCmvMapped: |
|---|
| 1894 | print 'INFO.isd005.0002: mapping vid: ',vid,extRefCmvMapped[vid] |
|---|
| 1895 | vid = extRefCmvMapped[vid] |
|---|
| 1896 | |
|---|
| 1897 | print 'INFO.isd005.0001: ',thisl, uid, vgid |
|---|
| 1898 | item.setAttribute( 'title', thisl + ' ((isd.005))' ) |
|---|
| 1899 | item.setAttribute( 'uid', thisuuid ) |
|---|
| 1900 | item.setAttribute( 'vgid', vgid ) |
|---|
| 1901 | item.setAttribute( 'vid', vid ) |
|---|
| 1902 | |
|---|
| 1903 | thisd['title'] = thisl + ' ((isd.005))' |
|---|
| 1904 | thisd['uid'] = thisuuid |
|---|
| 1905 | if vid in self.cmvMap002: |
|---|
| 1906 | print 'INFO.vid.00001: resetting requestvar vid: %s ---> %s' % (vid,self.cmvMap002[vid]) |
|---|
| 1907 | vid = self.cmvMap002[vid] |
|---|
| 1908 | thisd['vid'] = vid |
|---|
| 1909 | thisd['vgid'] = vgid |
|---|
| 1910 | |
|---|
| 1911 | ################################## |
|---|
| 1912 | elif k == 'requestlink': |
|---|
| 1913 | ## - uuid; mip; tab; objective; grid; gridreq; comment |
|---|
| 1914 | kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'ref', 'refNote', 'refid'] |
|---|
| 1915 | |
|---|
| 1916 | uid = str( ll[0] ) |
|---|
| 1917 | mip = string.strip(str( ll[1] ) ) |
|---|
| 1918 | obj = string.strip(str( ll[3] ) ) |
|---|
| 1919 | if mip == "GMMIP": |
|---|
| 1920 | bits = [ mycc(x) for x in string.split( obj, ' ' ) ] |
|---|
| 1921 | else: |
|---|
| 1922 | bits = [ mycc(x) for x in string.split( obj, ',' ) ] |
|---|
| 1923 | |
|---|
| 1924 | ## |
|---|
| 1925 | ## look for objectives specified in experiment column .... |
|---|
| 1926 | ## |
|---|
| 1927 | |
|---|
| 1928 | for b in bits: |
|---|
| 1929 | self.objectiveLinks[mip].a[b].append( str( ll[0] ) ) |
|---|
| 1930 | self.requestlinkuid.add( str(ll[0] ) ) |
|---|
| 1931 | self.rqlPreset[str(ll[0])] = ll[9] |
|---|
| 1932 | ttl = string.strip(str( ll[2] ) ) |
|---|
| 1933 | self.rqLnks[str(ll[0])] = ll[:] |
|---|
| 1934 | dothis = False |
|---|
| 1935 | |
|---|
| 1936 | ################################## |
|---|
| 1937 | elif k == 'requestitem': |
|---|
| 1938 | ## - mip; tab; expt; rlid; ny |
|---|
| 1939 | ## defer so that cross-check of "expt" can be done .. or scan this earlier .... |
|---|
| 1940 | kk = ['mip', 'tab', 'expt', 'rlid', 'ny', 'nexmax', 'nenmax', 'nymax','treset','info'] |
|---|
| 1941 | finishedClones = False |
|---|
| 1942 | ku = str(ll[2]) |
|---|
| 1943 | if self.exptPntUid[ku][1] == 'list' and len( self.exptPntUid[ku][2] ) > 0: |
|---|
| 1944 | cloning = True |
|---|
| 1945 | clones = set( self.exptPntUid[ku][2] ) |
|---|
| 1946 | kcl = 0 |
|---|
| 1947 | else: |
|---|
| 1948 | cloning = False |
|---|
| 1949 | |
|---|
| 1950 | while not finishedClones: |
|---|
| 1951 | thisl = mycc( '%s %s' % (ll[0],ll[1]) ) |
|---|
| 1952 | if str( ll[3] ) not in self.rqlPreset: |
|---|
| 1953 | print 'ERROR.090.02000: requestlink not found .. ',ll |
|---|
| 1954 | preset = '-99' |
|---|
| 1955 | else: |
|---|
| 1956 | preset = str( self.rqlPreset[ str( ll[3] ) ] ) |
|---|
| 1957 | if str( ll[3] ) not in self.requestlinkuid: |
|---|
| 1958 | print 'SEVERE.005.00005: rlid not found: %s' % str(ll) |
|---|
| 1959 | for j in range(len(kk)-1): |
|---|
| 1960 | if kk[j] == 'ny': |
|---|
| 1961 | if ll[j] == '': |
|---|
| 1962 | thisv = 0 |
|---|
| 1963 | else: |
|---|
| 1964 | try: |
|---|
| 1965 | thisv = str( int(ll[j]) ) |
|---|
| 1966 | except: |
|---|
| 1967 | print 'SEVERE: failed to set "ny": %s' % str(ll) |
|---|
| 1968 | thisv = 999 |
|---|
| 1969 | else: |
|---|
| 1970 | thisv = str( ll[j] ) |
|---|
| 1971 | if not (kk[j] == 'treset' and thisv in ['None','']): |
|---|
| 1972 | item.setAttribute( kk[j], thisv ) |
|---|
| 1973 | item.setAttribute( 'title', '%s, %s, %s' % (ll[0],ll[1],ll[2]) ) |
|---|
| 1974 | item.setAttribute( 'uid', str(i) ) |
|---|
| 1975 | item.setAttribute( 'preset', preset ) |
|---|
| 1976 | if cloning: |
|---|
| 1977 | ku = clones.pop() |
|---|
| 1978 | if kcl != 0: |
|---|
| 1979 | item.setAttribute( 'uid', str(uuid.uuid1()) ) |
|---|
| 1980 | kcl += 1 |
|---|
| 1981 | ##print 'INFO.clone.00001: ',ku, kcl |
|---|
| 1982 | if ku in self.experiments: |
|---|
| 1983 | esid = self.experiments[ku] |
|---|
| 1984 | esidmsg = 'Experiment %s' % ku |
|---|
| 1985 | elif ku in self.exptPntUid: |
|---|
| 1986 | esid = self.exptPntUid[ku][0] |
|---|
| 1987 | esidmsg = str(self.exptPntUid[ku]) |
|---|
| 1988 | else: |
|---|
| 1989 | print 'ERROR.esid.00001:', ku, ' Not Found' |
|---|
| 1990 | esidmsg = 'ERROR: Experiment %s Not Found' % ku |
|---|
| 1991 | esid = str(uuid.uuid1()) |
|---|
| 1992 | |
|---|
| 1993 | item.setAttribute( 'esid', esid ) |
|---|
| 1994 | item.setAttribute( 'esidComment', esidmsg ) |
|---|
| 1995 | if type(ll[9]) in [type(''), type(u'')]: |
|---|
| 1996 | kkk = ll[9].strip( ) |
|---|
| 1997 | if kkk[:3] == '_sl': |
|---|
| 1998 | item.setAttribute( 'tslice', kkk ) |
|---|
| 1999 | elif kkk not in ['','all']: |
|---|
| 2000 | print 'ERROR.tslice.0007: unrecognised slice: %s' % kkk |
|---|
| 2001 | |
|---|
| 2002 | finishedClones = (not cloning) or ( len( clones) == 0 ) |
|---|
| 2003 | if not finishedClones: |
|---|
| 2004 | ff[k][0].appendChild( item ) |
|---|
| 2005 | item = self.doc.createElement( 'item' ) |
|---|
| 2006 | ################################## |
|---|
| 2007 | elif k in ['requestvargroup','tablesection']: |
|---|
| 2008 | if k == 'requestvargroup': |
|---|
| 2009 | kk = ['uid', 'mip', 'label', 'title', 'ref', 'refNote'] |
|---|
| 2010 | else: |
|---|
| 2011 | kk = ['uid', 'gpid', 'mip', 'label', 'title', 'ref', 'refNote'] |
|---|
| 2012 | thisl = string.replace( str( ll[2] ), '.', '-' ) |
|---|
| 2013 | omitrvg = {'C4MIP-Omon','C4MIP-Oclim','HighResMIP-OImon','HighResMIP-Oclim'} |
|---|
| 2014 | if thisl in omitrvg: |
|---|
| 2015 | dothis = False |
|---|
| 2016 | else: |
|---|
| 2017 | thisd = {} |
|---|
| 2018 | for j in range(len(kk)): |
|---|
| 2019 | item.setAttribute( kk[j], str( ll[j] ) ) |
|---|
| 2020 | thisd[kk[j]] = str( ll[j] ) |
|---|
| 2021 | #### |
|---|
| 2022 | if dothis: |
|---|
| 2023 | if labmod: |
|---|
| 2024 | if string.find( thisl, '_' ) != -1: |
|---|
| 2025 | thisl = string.replace( thisl, '_', '-' ) |
|---|
| 2026 | if string.find( thisl, '*' ) != -1: |
|---|
| 2027 | thisl = string.replace( thisl, '*', '-' ) |
|---|
| 2028 | if string.find( thisl, '!' ) != -1: |
|---|
| 2029 | thisl = string.replace( thisl, '!', '-' ) |
|---|
| 2030 | if string.find( thisl, ' ' ) != -1: |
|---|
| 2031 | thisl = string.replace( thisl, ' ', '-' ) |
|---|
| 2032 | if string.find( thisl, ':' ) != -1: |
|---|
| 2033 | thisl = string.replace( thisl, ':', '-' ) |
|---|
| 2034 | item.setAttribute( 'label', thisl ) |
|---|
| 2035 | ff[k][0].appendChild( item ) |
|---|
| 2036 | if k in ['requestvargroup','tablesection']: |
|---|
| 2037 | thisd['label'] = thisl |
|---|
| 2038 | self.rqvgList.append( thisd ) |
|---|
| 2039 | if len(nfr) > 0: |
|---|
| 2040 | print 'ERROR:nfr: ',nfr |
|---|
| 2041 | |
|---|
| 2042 | def fixfixcmv(self): |
|---|
| 2043 | """deal with changing cmv ids relative to fixcmv""" |
|---|
| 2044 | ##self.cmvinfo = ('uid','stid','label','title','description','vid', \ |
|---|
| 2045 | ##'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', \ |
|---|
| 2046 | ##'deflate', 'valid_min', 'flag_values', 'modeling_realm', 'valid_max', 'positive', 'mipTable', \ |
|---|
| 2047 | ##'prov', 'provNote', 'frequency', 'rowIndex', 'coordinates','defaultPriority','mipTableSection') |
|---|
| 2048 | ff = {} |
|---|
| 2049 | for v in self.cmv: |
|---|
| 2050 | kkk = '%s.%s.%s' % (v[2],v[18],v[19]) |
|---|
| 2051 | if kkk in self.fix.fixcmvx: |
|---|
| 2052 | for k in self.fix.fixcmvx[kkk]: |
|---|
| 2053 | if k != v[0]: |
|---|
| 2054 | print 'WARN.cmv.00040: changed uid:', k, v |
|---|
| 2055 | ff[k] = v[0] |
|---|
| 2056 | ## ['label','uid','table','mode','uid2','prov'] |
|---|
| 2057 | |
|---|
| 2058 | for vid in self.fix.fixcmv.keys(): |
|---|
| 2059 | r = list( self.fix.fixcmv[vid] ) |
|---|
| 2060 | print r |
|---|
| 2061 | if r[1] in ff or r[4] in ff: |
|---|
| 2062 | if r[1] in ff: |
|---|
| 2063 | r[1] = ff[r[1]] |
|---|
| 2064 | if r[4] in ff: |
|---|
| 2065 | r[4] = ff[r[4]] |
|---|
| 2066 | self.fix.fixcmv[r[1]] = r |
|---|
| 2067 | |
|---|
| 2068 | |
|---|
| 2069 | def write2(self): |
|---|
| 2070 | """write sections for cmor variable, request variable, structure, spatial and temporal dimensions""" |
|---|
| 2071 | spsResets = collections.defaultdict(list) |
|---|
| 2072 | doneStructures = set() |
|---|
| 2073 | doneMessages = set() |
|---|
| 2074 | self.fixfixcmv() |
|---|
| 2075 | self.spLab2Uid = {} |
|---|
| 2076 | self.tmLab2Uid = {} |
|---|
| 2077 | self.tmUid2Lab = {} |
|---|
| 2078 | self.tmsp2str = {} |
|---|
| 2079 | self.tables = set() |
|---|
| 2080 | for ksect in [ u'spatialshape', u'temporalshape', u'structure', u'cmorvar', u'requestvar', u'requestlink']: |
|---|
| 2081 | ##for ksect in [ u'cmorvar', u'spatialshape', u'temporalshape', u'structure']: |
|---|
| 2082 | dil = self.ff[ksect][0].getElementsByTagName('item') |
|---|
| 2083 | if ksect == 'requestvar': |
|---|
| 2084 | for item in dil: |
|---|
| 2085 | vid = item.getAttribute( 'vid' ) |
|---|
| 2086 | if vid in self.fix.fixcmv: |
|---|
| 2087 | if self.fix.fixcmv[vid][3] == u'replace': |
|---|
| 2088 | vid1 = str( self.fix.fixcmv[vid][4] ) |
|---|
| 2089 | if vid1 not in self.cmvByUid: |
|---|
| 2090 | print 'ERROR.fixcmv.0002: attempt to use invalid vid: ',item.getAttribute( 'label' ),vid1,vid in self.cmvByUid |
|---|
| 2091 | else: |
|---|
| 2092 | item.setAttribute( 'vid', vid1 ) |
|---|
| 2093 | print 'INFO.fixcmv.00020: replacing link in requestVar %s: %s' % (item.getAttribute( 'uid' ),vid1) |
|---|
| 2094 | else: |
|---|
| 2095 | for d in dil: |
|---|
| 2096 | self.ff[ksect][0].removeChild(d) |
|---|
| 2097 | |
|---|
| 2098 | if ksect == u'cmorvar': |
|---|
| 2099 | ktable = self.cmvinfo.index('mipTable') |
|---|
| 2100 | kfreq = self.cmvinfo.index('frequency') |
|---|
| 2101 | klabel = self.cmvinfo.index('label') |
|---|
| 2102 | |
|---|
| 2103 | s1 = dict() |
|---|
| 2104 | stab = set() |
|---|
| 2105 | tmidpt = self.tmLab2Uid['time-point'] |
|---|
| 2106 | self._cmvLabByUid = {} |
|---|
| 2107 | for r in self.cmv: |
|---|
| 2108 | if r[0] in self.rep['cmv'].items: |
|---|
| 2109 | print 'INFO.090.09099: detected cmv replace request: ',r |
|---|
| 2110 | elif r[0] in s1: |
|---|
| 2111 | print 'ERROR.090.09005: duplicate key at ',r,s1[r[0]] |
|---|
| 2112 | else: |
|---|
| 2113 | s1[ r[0] ] = (r[ktable],r[kfreq],r[klabel]) |
|---|
| 2114 | stab.add( r[ktable] ) |
|---|
| 2115 | u0 = r[0] |
|---|
| 2116 | l0 = r[2] |
|---|
| 2117 | # |
|---|
| 2118 | # identify structures for AerChemMIP tables |
|---|
| 2119 | # |
|---|
| 2120 | if len( r[1] ) == 2: |
|---|
| 2121 | spid = self.spLab2Uid[r[1][0]] |
|---|
| 2122 | if r[1][1] == 'mean': |
|---|
| 2123 | tmid = self.tmLab2Uid['time-mean'] |
|---|
| 2124 | tmode = 'mean' |
|---|
| 2125 | else: |
|---|
| 2126 | tmid = self.tmLab2Uid['time-point'] |
|---|
| 2127 | tmode = 'point' |
|---|
| 2128 | |
|---|
| 2129 | if (tmid,spid,tmode) in self.lls.tmsp2str: |
|---|
| 2130 | stid = self.lls.tmsp2str[ (tmid,spid,tmode) ] |
|---|
| 2131 | else: |
|---|
| 2132 | print 'SEVERE.stid.0001: no structure record for ',tmid,spid,tmode,r[1][0] |
|---|
| 2133 | stid = '__stid_not_found__' |
|---|
| 2134 | print 'INFO.klklkl: AerChemMIP structure .... ',r[1], spid |
|---|
| 2135 | else: |
|---|
| 2136 | if r[1] in self.strdd: |
|---|
| 2137 | stid = self.strdd[r[1]] |
|---|
| 2138 | else: |
|---|
| 2139 | for k in self.strdd.keys(): |
|---|
| 2140 | if k[0] == r[1][0]: |
|---|
| 2141 | print '*** ',k |
|---|
| 2142 | else: |
|---|
| 2143 | print '+++ ',k |
|---|
| 2144 | print 'SEVERE.999.00099: strdd key not found: %s' % str(r[1]) |
|---|
| 2145 | print r |
|---|
| 2146 | stid = '__struct_not_found_001__' |
|---|
| 2147 | |
|---|
| 2148 | stinfo = self.structInfo.get( stid, ['','','',''] ) |
|---|
| 2149 | spinfo = self.spidInfo.get( stinfo[2] , ['','','','',''] ) |
|---|
| 2150 | tmid = stinfo[3] |
|---|
| 2151 | if "plev7h" in spinfo[3]: |
|---|
| 2152 | lsfx = '7h' |
|---|
| 2153 | elif "plev7c" in spinfo[3]: |
|---|
| 2154 | lsfx = '' |
|---|
| 2155 | elif "plev4" in spinfo[3]: |
|---|
| 2156 | lsfx = '4' |
|---|
| 2157 | elif "plev23" in spinfo[3] and r[2] == 'tntlw' : |
|---|
| 2158 | ##print 'plev7h found',r,spinfo |
|---|
| 2159 | lsfx = '23' |
|---|
| 2160 | else: |
|---|
| 2161 | lsfx = '' |
|---|
| 2162 | |
|---|
| 2163 | if r[2] == 'clisccp': |
|---|
| 2164 | print 'INFO.clisccp.00001:',r |
|---|
| 2165 | |
|---|
| 2166 | if r[0] in self.fix.fixcmv: |
|---|
| 2167 | ## print 'INFO.fixcmv.00010: omitting record %s, %s, %s' % (str( self.fix.fixcmv[r[0]] ), r[0],r[2] ) |
|---|
| 2168 | pass |
|---|
| 2169 | else: |
|---|
| 2170 | item = self.doc.createElement( 'item' ) |
|---|
| 2171 | for k in range(len(self.cmvinfo)): |
|---|
| 2172 | n = self.cmvinfo[k] |
|---|
| 2173 | if n in self.cmvinfo2: |
|---|
| 2174 | if n == 'mipTable' and str(r[k]) == 'CMIP5_Oyr': |
|---|
| 2175 | print 'ERROR.047.0002: ',r |
|---|
| 2176 | if n == "vid": |
|---|
| 2177 | if str(r[k]) == '__var_not_found_012__': |
|---|
| 2178 | print 'ERROR.099.00202: bad vid string: %s [%s]' % (r[k],str(r[:3])) |
|---|
| 2179 | vid = str(r[k]) |
|---|
| 2180 | if vid in self.importWbMods.vars.vmap: |
|---|
| 2181 | ov = vid |
|---|
| 2182 | vid = self.importWbMods.vars.vmap[vid] |
|---|
| 2183 | print 'INFO.040.04444: mapping',ov,vid |
|---|
| 2184 | |
|---|
| 2185 | if n == "rowIndex": |
|---|
| 2186 | ix = str(r[k]) |
|---|
| 2187 | elif n == "defaultPriority": |
|---|
| 2188 | priority = str(r[k]) |
|---|
| 2189 | elif n == "title": |
|---|
| 2190 | title = str(r[k]) |
|---|
| 2191 | |
|---|
| 2192 | if n == "uid": |
|---|
| 2193 | if r[k].find( 'ea313ee8-a0de-11e6-bc63-ac72891c3257' ) != -1: |
|---|
| 2194 | print 'INFO.uid.9999: ',r |
|---|
| 2195 | item.setAttribute( n, r[k].strip() ) |
|---|
| 2196 | elif n == "stid": |
|---|
| 2197 | item.setAttribute( n, stid ) |
|---|
| 2198 | elif n in ["mipTableSection","processing"]: |
|---|
| 2199 | if str(r[k]) != '': |
|---|
| 2200 | item.setAttribute( n, str(r[k]) ) |
|---|
| 2201 | elif n == "label": |
|---|
| 2202 | thisl = string.replace( r[k], '_', '' ) |
|---|
| 2203 | if len(lsfx) > 0: |
|---|
| 2204 | if thisl[ - len( lsfx): ] != lsfx: |
|---|
| 2205 | thisl += lsfx |
|---|
| 2206 | item.setAttribute( n, thisl ) |
|---|
| 2207 | elif n in ["valid_min","valid_max","ok_max_mean_abs","ok_min_mean_abs"]: |
|---|
| 2208 | if str(r[k]) != "" and r[k] not in [None, 'None']: |
|---|
| 2209 | item.setAttribute( n, str(r[k]) ) |
|---|
| 2210 | elif n == "frequency": |
|---|
| 2211 | fr = freqmap.get( str(r[k]), str(r[k]) ) |
|---|
| 2212 | item.setAttribute( n, fr ) |
|---|
| 2213 | elif n == "mipTable": |
|---|
| 2214 | tn = tableNameMap.get( str(r[k]), str(r[k]) ) |
|---|
| 2215 | if tn == 'em6hr': |
|---|
| 2216 | if 'alev' in spinfo[3]: |
|---|
| 2217 | tn = '6hrLev' |
|---|
| 2218 | else: |
|---|
| 2219 | if tmid == tmidpt: |
|---|
| 2220 | tn = '6hrPlevpt' |
|---|
| 2221 | else: |
|---|
| 2222 | tn = '6hrPlev' |
|---|
| 2223 | print 'INFO.6hrPlev.0001:', tmid, tmidpt, tmid == tmidpt, r,tn, self.tmUid2Lab[tmid] |
|---|
| 2224 | item.setAttribute( n, tn ) |
|---|
| 2225 | else: |
|---|
| 2226 | item.setAttribute( n, str(r[k]) ) |
|---|
| 2227 | |
|---|
| 2228 | ### reset table name for OImon ... correcting for frequency as well |
|---|
| 2229 | ### |
|---|
| 2230 | omitThis = False |
|---|
| 2231 | mtable = item.getAttribute( 'mipTable' ) |
|---|
| 2232 | prov = item.getAttribute( 'prov' ) |
|---|
| 2233 | realm = item.getAttribute( 'modeling_realm' ) |
|---|
| 2234 | if realm in ['','__unset__']: |
|---|
| 2235 | if prov in knownRealmA: |
|---|
| 2236 | item.setAttribute( 'modeling_realm', 'atmos' ) |
|---|
| 2237 | elif prov in knownRealmO: |
|---|
| 2238 | item.setAttribute( 'modeling_realm', 'ocean' ) |
|---|
| 2239 | |
|---|
| 2240 | mtable0 = mtable |
|---|
| 2241 | if mtable0 == 'OImon': |
|---|
| 2242 | if r[kfreq] == 'mon': |
|---|
| 2243 | mtable = 'SImon' |
|---|
| 2244 | else: |
|---|
| 2245 | mtable = 'SIday' |
|---|
| 2246 | elif mtable0 in ['em6hr','em3hr','emDay','emMon']: |
|---|
| 2247 | if stid in self.structInfo: |
|---|
| 2248 | if string.find( self.structInfo[stid][1], 'time: point' ) != -1 or tmid == tmidpt: |
|---|
| 2249 | mtable += 'pt' |
|---|
| 2250 | if string.find( self.spatialDims[ self.structInfo[stid][2] ], 'longitude' ) == -1 and mtable[-1] != 'Z': |
|---|
| 2251 | mtable += 'Z' |
|---|
| 2252 | else: |
|---|
| 2253 | print 'WARN.090.09999: structure uid not found',stid, r |
|---|
| 2254 | elif mtable0 == 'Amon' and thisl[-6:] == 'Adjust': |
|---|
| 2255 | print 'INFO.adjust.0001: %s moved to AmonAdj .. and omitted' % thisl |
|---|
| 2256 | mtable = 'AmonAdj' |
|---|
| 2257 | omitThis = True |
|---|
| 2258 | if mtable != r[ktable]: |
|---|
| 2259 | item.setAttribute( 'mipTable', mtable ) |
|---|
| 2260 | item.setAttribute( 'mtid', 'MIPtable::%s' % mtable ) |
|---|
| 2261 | self.tables.add( mtable ) |
|---|
| 2262 | |
|---|
| 2263 | thisuid = r[0] |
|---|
| 2264 | if tableUtils.getSubGroup(spinfo[3],mtable,thisl,ix,title,thisuid,priority): |
|---|
| 2265 | item.setAttribute( 'subGroup', tableUtils.subGroup ) |
|---|
| 2266 | |
|---|
| 2267 | if str(vid) not in self.importWbMods.vars.d1: |
|---|
| 2268 | |
|---|
| 2269 | if thisl in self.importWbMods.vars.c1 or (thisl[-2:] in ['17','19','23','27','7c','7h'] and thisl[:-2] in self.importWbMods.vars.c1): |
|---|
| 2270 | if thisl in self.importWbMods.vars.c1: |
|---|
| 2271 | thisn = thisl |
|---|
| 2272 | else: |
|---|
| 2273 | thisn = thisl[:-2] |
|---|
| 2274 | |
|---|
| 2275 | if len( self.importWbMods.vars.c1[ thisn ] ) == 1: |
|---|
| 2276 | newvid = self.importWbMods.vars.c1[ thisn ][0] |
|---|
| 2277 | item.setAttribute( 'vid', newvid ) |
|---|
| 2278 | print 'INFO.099.00110: bad vid string replaced: %s [%s]' % (newvid,thisl) |
|---|
| 2279 | else: |
|---|
| 2280 | print 'ERROR.099.00111: ambiguous bad vid string: %s [%s]' % (vid,thisl) |
|---|
| 2281 | else: |
|---|
| 2282 | print 'ERROR.099.00112: bad vid string: %s [%s]' % (vid,thisl) |
|---|
| 2283 | elif thisl == 'H2p': |
|---|
| 2284 | print 'WARN.h2p.00001: H2p vid recording as OK here .... %s' % vid |
|---|
| 2285 | if self.repl.has_key( thisuid ): |
|---|
| 2286 | for tag,old,new in self.repl[thisuid]: |
|---|
| 2287 | thisold = item.getAttribute( tag ) |
|---|
| 2288 | if thisold != old: |
|---|
| 2289 | ##print 'WARN.yyy: Attempt to replace value which is not present' |
|---|
| 2290 | print 'WARN.050.00051: Failed attempt to replace values in cmv: %s old:%s new:%s' % (tag,old,new) |
|---|
| 2291 | else: |
|---|
| 2292 | if tag == 'vid' and new not in self.importWbMods.vars.d1: |
|---|
| 2293 | print 'ERROR.050.00052: Attempt to replave values in cmv with invalid vid: %s old:%s new:%s' % (tag,old,new) |
|---|
| 2294 | else: |
|---|
| 2295 | item.setAttribute( tag, new ) |
|---|
| 2296 | print 'INFO.050.00050: replacing values in cmv: %s old:%s new:%s' % (tag,old,new) |
|---|
| 2297 | |
|---|
| 2298 | if thisuid in self.deferredLabMap: |
|---|
| 2299 | lab0,lab1 = self.deferredLabMap[thisuid] |
|---|
| 2300 | labold = item.getAttribute( 'label' ) |
|---|
| 2301 | if labold != lab0: |
|---|
| 2302 | print 'ERROR.labmap.0001: attempt to change wrong label: ',lab0,lab1,labold |
|---|
| 2303 | else: |
|---|
| 2304 | item.setAttribute( 'label', lab1 ) |
|---|
| 2305 | print 'INFO.labmap.0001: attempt to change label: ',lab0,lab1 |
|---|
| 2306 | |
|---|
| 2307 | ### final check on structure .. but won't work if tuple not found ... but why is structure not there?? |
|---|
| 2308 | stid = item.getAttribute( 'stid' ) |
|---|
| 2309 | if stid in self.struidmapextra: |
|---|
| 2310 | mid = ('INFO.structure2',stid) |
|---|
| 2311 | if mid not in doneMessages: |
|---|
| 2312 | print 'INFO.structure: mapping [2]: %s to %s' % (stid,self.struidmapextra[stid]) |
|---|
| 2313 | doneMessages.add(mid) |
|---|
| 2314 | item.setAttribute( 'stid', self.struidmapextra[stid] ) |
|---|
| 2315 | stid = self.struidmapextra[stid] |
|---|
| 2316 | if stid in WbMods.str.uidMap: |
|---|
| 2317 | mid = ('INFO.structure',stid) |
|---|
| 2318 | if mid not in doneMessages: |
|---|
| 2319 | print 'INFO.structure: mapping: %s to %s' % (stid,WbMods.str.uidMap[stid]) |
|---|
| 2320 | doneMessages.add(mid) |
|---|
| 2321 | if stid in self.struidmapextra: |
|---|
| 2322 | assert self.struidmapextra[stid] == WbMods.str.uidMap[stid], 'CONFLICT in stid mappings: %s, %s, %s' % (stid,self.struidmapextra[stid], WbMods.str.uidMap[stid]) |
|---|
| 2323 | item.setAttribute( 'stid', WbMods.str.uidMap[stid] ) |
|---|
| 2324 | |
|---|
| 2325 | ## append CMORvar record |
|---|
| 2326 | u1 = item.getAttribute('uid') |
|---|
| 2327 | if u1 in extRefCmvMapped: |
|---|
| 2328 | print 'WARN.uidmap.00004: mapping CMORvar uid %s --> %s' % (u1,extRefCmvMapped[u1] ) |
|---|
| 2329 | item.setAttribute('uid',extRefCmvMapped[u1]) |
|---|
| 2330 | l1 = item.getAttribute( 'label') |
|---|
| 2331 | #up: rlus rlut rlutcs hfls hfss mcu rsus rsut rsutcs down:rlds rsds |
|---|
| 2332 | if l1 in varPositive: |
|---|
| 2333 | item.setAttribute( 'positive', varPositive[l1] ) |
|---|
| 2334 | print 'INFO.positive.00001: setting positive [%s]: %s' % (l1,varPositive[l1]) |
|---|
| 2335 | if l1 != l0: |
|---|
| 2336 | print 'WARN.cmvlab.00030: changed cmv label: %s %s %s %s' % (l0,l1,u0,u1) |
|---|
| 2337 | self._cmvLabByUid[item.getAttribute('uid')] = item.getAttribute( 'label') |
|---|
| 2338 | if not omitThis: |
|---|
| 2339 | self.ff[ksect][0].appendChild( item ) |
|---|
| 2340 | print 'INFO.tables.0001: ',stab |
|---|
| 2341 | |
|---|
| 2342 | elif ksect == u'structure': |
|---|
| 2343 | ers = {} |
|---|
| 2344 | for kk in spsResets: |
|---|
| 2345 | if len( spsResets[kk] ) > 0: |
|---|
| 2346 | utarg = spsResets[kk][0][0] |
|---|
| 2347 | for r in spsResets[kk][1:]: |
|---|
| 2348 | ers[r[0]] = utarg |
|---|
| 2349 | ##flds = self.structList[0][1]._fields |
|---|
| 2350 | flds = self.lls.structAttr |
|---|
| 2351 | for u,nt in self.lls.ll: |
|---|
| 2352 | smode = 'new' |
|---|
| 2353 | if smode == 'new': |
|---|
| 2354 | d1 = self.lls.asDict( u ) |
|---|
| 2355 | tlab = d1['time label'] |
|---|
| 2356 | ##print 'INFO.stid.00072: %s, %s' % (u,str( d1 )) |
|---|
| 2357 | else: |
|---|
| 2358 | ##print 'INFO.stid.00071: %s, %s' % (u,str( nt.__dict__ )) |
|---|
| 2359 | d1 = {} |
|---|
| 2360 | for f in flds: |
|---|
| 2361 | if f == 'spid' and nt.__dict__[f] in ers: |
|---|
| 2362 | d1[f] = ers[ nt.__dict__[f] ] |
|---|
| 2363 | print 'INFO.801.00011: updating target spid .... ', d1[f] |
|---|
| 2364 | elif f == 'coords' and 't2000' in nt.__dict__[f]: |
|---|
| 2365 | d1[f] = str( nt.__dict__[f] ) |
|---|
| 2366 | elif f == 'coords' and nt.__dict__[f][:4] in ['plev']: |
|---|
| 2367 | cnew = 'p' + nt.__dict__[f][4:] |
|---|
| 2368 | d1[f] = str( cnew ) |
|---|
| 2369 | |
|---|
| 2370 | print 'INFO.801.00013: updating target coords .... %s --> %s ' % ( nt.__dict__[f], cnew ) |
|---|
| 2371 | elif f == 'uid': |
|---|
| 2372 | d1[f] = u |
|---|
| 2373 | elif f == 'tlab': |
|---|
| 2374 | tlab = nt.tlab |
|---|
| 2375 | tmid = self.tmLab2Uid[tlab] |
|---|
| 2376 | d1['tmid'] = tmid |
|---|
| 2377 | |
|---|
| 2378 | else: |
|---|
| 2379 | d1[f] = str( nt.__dict__[f] ) |
|---|
| 2380 | if d1.get( 'cell_measures', '' ) == '@OPT': |
|---|
| 2381 | print 'WARN.structure.00077: changing cell_measures @OPT to --OPT',str(d1 ) |
|---|
| 2382 | d1['cell_measures'] = '--OPT' |
|---|
| 2383 | |
|---|
| 2384 | if d1['spid'] in WbMods.ls.labByUid: |
|---|
| 2385 | slab = WbMods.ls.labByUid[ d1['spid'] ] |
|---|
| 2386 | print 'INFO.801.00061: spid found in structure record ... %s\n%s' % (d1['spid'],slab) |
|---|
| 2387 | else: |
|---|
| 2388 | slab = '?' |
|---|
| 2389 | print 'SEVERE.801.00060: bad spid found in structure record ... %s\n%s' % (d1['spid'],str(d1)) |
|---|
| 2390 | cm = d1['cell_methods'] |
|---|
| 2391 | if cm == 'null': |
|---|
| 2392 | if tlab == 'time-point': |
|---|
| 2393 | cm = 'time: point' |
|---|
| 2394 | d1['cell_methods'] = cm |
|---|
| 2395 | elif tlab == 'time-mean': |
|---|
| 2396 | cm = 'time: mean' |
|---|
| 2397 | d1['cell_methods'] = cm |
|---|
| 2398 | elif cm in WbMods.cellm.cmmap: |
|---|
| 2399 | cm0 = cm |
|---|
| 2400 | cm = WbMods.cellm.cmmap[cm] |
|---|
| 2401 | print 'INFO.cmmap.00001: ',cm0,cm |
|---|
| 2402 | d1['cell_methods'] = cm |
|---|
| 2403 | |
|---|
| 2404 | cc = d1['coords'] |
|---|
| 2405 | oo = d1['odims'] |
|---|
| 2406 | if cc not in ['','?']: |
|---|
| 2407 | print 'INFO.coords.00001: >%s<' % cc |
|---|
| 2408 | d1['cids'] = 'dim:%s' % cc |
|---|
| 2409 | if oo not in ['','?']: |
|---|
| 2410 | ooc = string.join( ['dim:%s' % x for x in oo.split( '|' )] ) |
|---|
| 2411 | print 'INFO.odims.00001: >%s<' % ooc |
|---|
| 2412 | d1['cids'] = ooc |
|---|
| 2413 | t = (tlab,slab,d1['odims'],d1['coords'],d1['cell_methods'],d1['cell_measures'],d1['flag_meanings']) |
|---|
| 2414 | |
|---|
| 2415 | dothis = True |
|---|
| 2416 | if t not in WbMods.str.lookup: |
|---|
| 2417 | print 'WARN.structure.0040: tuple not in lookup: ',t,d1['uid'],d1['spid'],d1 |
|---|
| 2418 | else: |
|---|
| 2419 | print 'WARN.structure.0041: tuple in lookup ignored: ',t,d1['uid'],d1['spid'],d1 |
|---|
| 2420 | lab = d1['label'] |
|---|
| 2421 | if lab in WbMods.str.labMap: |
|---|
| 2422 | print 'INFO.structure.0050: omitting structure record ....',t,lab, d1['uid'] |
|---|
| 2423 | lab2 = WbMods.str.labMap[lab] |
|---|
| 2424 | dothis = False |
|---|
| 2425 | self.struidmapextra[d1['uid']] = WbMods.str.uidByLab[lab2] |
|---|
| 2426 | print 'INFO.structure.0062: setting stid mapping ....',t,d1['uid'],WbMods.str.uidByLab[lab2], d1['label'], lab2 |
|---|
| 2427 | |
|---|
| 2428 | lab = d1['label'] |
|---|
| 2429 | if lab in WbMods.str.mods: |
|---|
| 2430 | for k in WbMods.str.mods[lab]: |
|---|
| 2431 | print 'INFO.structure.0055: modifying attributes: %s: %s: %s' % (lab,k,WbMods.str.mods[lab][k]) |
|---|
| 2432 | d1[k] = WbMods.str.mods[lab][k] |
|---|
| 2433 | if k == 'cell_methods': |
|---|
| 2434 | cm = WbMods.str.mods[lab][k] |
|---|
| 2435 | if cm in WbMods.cellm.cminfo: |
|---|
| 2436 | d1['cmid'] = WbMods.cellm.cminfo[cm][2] |
|---|
| 2437 | else: |
|---|
| 2438 | print 'SEVERE.structure.0055: cmid for new cell method not found: %s' % (cm) |
|---|
| 2439 | |
|---|
| 2440 | if dothis and d1['uid'] in doneStructures: |
|---|
| 2441 | print 'WARN.structure.05021: repeat uid in structure definitions ....%s' % d1['uid'] |
|---|
| 2442 | if dothis and d1['uid'] not in doneStructures and d1['uid'] not in WbMods.str.uidMap: |
|---|
| 2443 | if cm not in WbMods.cellm.cminfo: |
|---|
| 2444 | print 'WARN.cellm.0001: Cell methods not found: ',cm, d1['uid'] |
|---|
| 2445 | else: |
|---|
| 2446 | d1['cmid'] = WbMods.cellm.cminfo[cm][2] |
|---|
| 2447 | |
|---|
| 2448 | doneStructures.add( d1['uid'] ) |
|---|
| 2449 | item = self.doc.createElement( 'item' ) |
|---|
| 2450 | for k in d1: |
|---|
| 2451 | if k not in ['time_label','spatial_label', 'time label','spatial label']: |
|---|
| 2452 | item.setAttribute( k.replace( ' ','_'), d1[k] ) |
|---|
| 2453 | self.ff[ksect][0].appendChild( item ) |
|---|
| 2454 | else: |
|---|
| 2455 | if not dothis: |
|---|
| 2456 | print 'INFO.structure.09001: omitting %s' % d1['label'] |
|---|
| 2457 | elif d1['uid'] in doneStructures: |
|---|
| 2458 | print 'INFO.structure.09002: omitting %s' % d1['label'] |
|---|
| 2459 | elif d1['uid'] in WbMods.str.uidMap: |
|---|
| 2460 | print 'INFO.structure.09003: omitting %s' % d1['label'] |
|---|
| 2461 | ### |
|---|
| 2462 | elif ksect == u'spatialshape': |
|---|
| 2463 | |
|---|
| 2464 | ## self.dimsSpaceInfo = ['uid','label','title','dimensions','levelFlag','levels'] |
|---|
| 2465 | self.spatialDims = {} |
|---|
| 2466 | for k in self.dimsSpace.keys(): |
|---|
| 2467 | rr = list( self.dimsSpace[k] ) |
|---|
| 2468 | kk = rr[3] |
|---|
| 2469 | if kk == 'scalar': |
|---|
| 2470 | kk = '' |
|---|
| 2471 | rr = list(rr) |
|---|
| 2472 | rr[3] = '' |
|---|
| 2473 | elif kk == 'longitude|latitude|plev': |
|---|
| 2474 | kk = 'longitude|latitude|plev4' |
|---|
| 2475 | rr = list(rr) |
|---|
| 2476 | rr[3] = kk |
|---|
| 2477 | elif kk == 'longitude|latitude|plev7': |
|---|
| 2478 | kk = 'longitude|latitude|plev7c' |
|---|
| 2479 | print 'INFO.plev7.00001: plev7 --> plev7c' |
|---|
| 2480 | rr = list(rr) |
|---|
| 2481 | rr[3] = kk |
|---|
| 2482 | self.spatialDims[rr[0]] = rr[3] |
|---|
| 2483 | bits = [x for x in rr[3].split( '|' ) if x not in ['','?']] |
|---|
| 2484 | rr.append( string.join( ['dim:%s' % x for x in bits] ) ) |
|---|
| 2485 | if kk not in spsResets: |
|---|
| 2486 | item = self.doc.createElement( 'item' ) |
|---|
| 2487 | for i in range(len(self.dimsSpaceInfo)): |
|---|
| 2488 | item.setAttribute( self.dimsSpaceInfo[i], str( rr[i] ) ) |
|---|
| 2489 | ##assert rr[3] in self.spsh.ss, 'FATAL: spatial shape record not recognised: %s' % str(rr) |
|---|
| 2490 | if rr[3] in self.spsh.ss: |
|---|
| 2491 | lab,ttl,lf,li,u = self.spsh.ss[rr[3]] |
|---|
| 2492 | if trace.SPATIALSHAPE_01: |
|---|
| 2493 | print 'INFO.035.0001: new spatial shape label: %s, %s' % (lab, ttl) |
|---|
| 2494 | if ttl[:4] != 'ssd.': |
|---|
| 2495 | item.setAttribute( 'title', ttl ) |
|---|
| 2496 | if len(lab) < 4 or lab[:4] != 'ssd-': |
|---|
| 2497 | item.setAttribute( 'label', lab ) |
|---|
| 2498 | item.setAttribute( 'levels', li ) |
|---|
| 2499 | item.setAttribute( 'uid', u ) |
|---|
| 2500 | item.setAttribute( 'levelFlag', string.lower(lf) ) |
|---|
| 2501 | else: |
|---|
| 2502 | print 'ERROR.055.00001: spatial shape record not recognised: %s' % str(rr) |
|---|
| 2503 | print 'INFO.801.00120: new shape for %s' % str(rr) |
|---|
| 2504 | thisl = item.getAttribute( 'label' ) |
|---|
| 2505 | assert thisl not in self.spLab2Uid.keys(), 'Duplicate spatial shape label: %s' % thisl |
|---|
| 2506 | self.spLab2Uid[thisl] = item.getAttribute( 'uid' ) |
|---|
| 2507 | self.ff[ksect][0].appendChild( item ) |
|---|
| 2508 | else: |
|---|
| 2509 | print 'INFO.801.00121: duplicate shape for %s' % str(rr) |
|---|
| 2510 | spsResets[kk].append( rr ) |
|---|
| 2511 | |
|---|
| 2512 | elif ksect == u'temporalshape': |
|---|
| 2513 | for k in self.dimsTime.keys(): |
|---|
| 2514 | rr = self.dimsTime[k] |
|---|
| 2515 | item = self.doc.createElement( 'item' ) |
|---|
| 2516 | for i in range(len(self.dimsTimeInfo)): |
|---|
| 2517 | item.setAttribute( self.dimsTimeInfo[i], str( rr[i] ) ) |
|---|
| 2518 | if self.dimsTimeInfo[i] == 'dimensions': |
|---|
| 2519 | item.setAttribute( 'dimid', 'dim:%s' % rr[i] ) |
|---|
| 2520 | self.ff[ksect][0].appendChild( item ) |
|---|
| 2521 | self.tmLab2Uid[ item.getAttribute( 'label' ) ] = item.getAttribute( 'uid' ) |
|---|
| 2522 | self.tmUid2Lab[ item.getAttribute( 'uid' ) ] = item.getAttribute( 'label' ) |
|---|
| 2523 | elif ksect == u'requestvar': |
|---|
| 2524 | ##kk = ['var', 'table', 'mip', 'vid', 'priority'] |
|---|
| 2525 | kk = ['label', 'table', 'mip', 'vid', 'priority','vgid','title','uid'] |
|---|
| 2526 | kktr = {'vgid':'gpid', 'title':'label'} |
|---|
| 2527 | kkf = 0 |
|---|
| 2528 | for t in self.rqv: |
|---|
| 2529 | if t[5][:6] == 'SPECS_': |
|---|
| 2530 | print 'SEVERE.specs.00001: mis-assigned mip: ',t |
|---|
| 2531 | if t[1][-6:] == 'Adjust': |
|---|
| 2532 | doThis = False |
|---|
| 2533 | else: |
|---|
| 2534 | item = self.doc.createElement( 'item' ) |
|---|
| 2535 | |
|---|
| 2536 | for i in range(len(kk)): |
|---|
| 2537 | k = kktr.get( kk[i], kk[i] ) |
|---|
| 2538 | v = t[ self.rqvinfo.index(k) ] |
|---|
| 2539 | if k == 'priority': |
|---|
| 2540 | item.setAttribute( kk[i], str( int(v) ) ) |
|---|
| 2541 | elif kk[i] == 'label': |
|---|
| 2542 | thisl = string.replace( v, '_', '' ) |
|---|
| 2543 | item.setAttribute( kk[i], thisl ) |
|---|
| 2544 | elif kk[i] == 'title': |
|---|
| 2545 | item.setAttribute( kk[i], str( v ) + ' ((isd.006))' ) |
|---|
| 2546 | elif kk[i] == 'table': |
|---|
| 2547 | pass |
|---|
| 2548 | else: |
|---|
| 2549 | item.setAttribute( kk[i], str( v ) ) |
|---|
| 2550 | |
|---|
| 2551 | vid = item.getAttribute( 'vid' ) |
|---|
| 2552 | if vid in self.fix.fixcmv: |
|---|
| 2553 | if self.fix.fixcmv[vid][3] == u'replace': |
|---|
| 2554 | vidn = str( self.fix.fixcmv[vid][4] ) |
|---|
| 2555 | if vidn not in self._cmvLabByUid: |
|---|
| 2556 | print 'SEVERE.fixcmv.00040: attempt to redirect request to non-existant variable: %s,%s,%s::%s' % (vid,vidn,thisl,str(self.fix.fixcmv[vid]) ) |
|---|
| 2557 | else: |
|---|
| 2558 | labn = self.cmvByUid[vidn][2] |
|---|
| 2559 | if vid in self.cmvByUid: |
|---|
| 2560 | lab = self.cmvByUid[vid][2] |
|---|
| 2561 | if lab != labn: |
|---|
| 2562 | print 'WARN.fixcmv.00001: label change: %s -- %s (%s --> %s)' % (lab,labn,vid,vidn) |
|---|
| 2563 | elif labn != thisl: |
|---|
| 2564 | print 'WARN.fixcmv.00002: label change [2]: %s -- %s (%s --> %s)' % (thisl,labn,vid,vidn) |
|---|
| 2565 | item.setAttribute( 'vid', vidn ) |
|---|
| 2566 | vid = vidn |
|---|
| 2567 | print 'INFO.fixcmv.00050: replacing link in requestVar %s %s %s' % (str(t),vidn,labn) |
|---|
| 2568 | |
|---|
| 2569 | newd = None |
|---|
| 2570 | if vid in self.importWbMods.mmsi.ss: |
|---|
| 2571 | print 'INFO.046.0002: mip map found: ', self.importWbMods.mmsi.ss[vid] |
|---|
| 2572 | mm = self.importWbMods.mmsi.ss[vid] |
|---|
| 2573 | ##newd = self._editRequestVar01( item, mm, t, {} ) |
|---|
| 2574 | |
|---|
| 2575 | vid = item.getAttribute( 'vid' ) |
|---|
| 2576 | if vid in self.cmvMap002: |
|---|
| 2577 | vid = self.cmvMap002[vid] |
|---|
| 2578 | item.setAttribute( 'vid', vid ) |
|---|
| 2579 | |
|---|
| 2580 | if vid in extRefCmvMapped: |
|---|
| 2581 | vid = extRefCmvMapped[vid] |
|---|
| 2582 | item.setAttribute( 'vid', vid ) |
|---|
| 2583 | |
|---|
| 2584 | if vid not in self.cmvByUid: |
|---|
| 2585 | print 'ERROR.reqvar.000001: vid not found: %s' % str(t) |
|---|
| 2586 | else: |
|---|
| 2587 | lab0 = self.cmvByUid[vid][2] |
|---|
| 2588 | if vid not in self._cmvLabByUid: |
|---|
| 2589 | print 'WARN.cmvlab.00020: no _cmv.. record ... %s %s: %s' % (vid,lab0,str( self.cmvByUid[vid] ) ) |
|---|
| 2590 | else: |
|---|
| 2591 | lab = self._cmvLabByUid[vid] |
|---|
| 2592 | if lab != lab0: |
|---|
| 2593 | print 'WARN.cmvlab.00010: difference in records ... %s %s: %s' % (vid,lab,str( self.cmvByUid[vid] ) ) |
|---|
| 2594 | if lab != thisl: |
|---|
| 2595 | ##print 'INFO.reqvar.00001: inconsistent requestvar label: %s --> %s (%s)' % (thisl, lab, vid) |
|---|
| 2596 | print 'INFO.reqvar.00001: changing requestvar label: %s --> %s' % (thisl, lab) |
|---|
| 2597 | item.setAttribute( 'label', lab ) |
|---|
| 2598 | |
|---|
| 2599 | tm = item.getAttribute( 'mip' ) |
|---|
| 2600 | |
|---|
| 2601 | did = '%s__%s' % (t[2],t[3]) |
|---|
| 2602 | if self.vgcheck.has_key(did): |
|---|
| 2603 | print 'ERROR.090.09002: Duplicate request ....: ',t,self.vgcheck[did] |
|---|
| 2604 | else: |
|---|
| 2605 | self.vgcheck[did] = t |
|---|
| 2606 | self.ff[ksect][0].appendChild( item ) |
|---|
| 2607 | ## |
|---|
| 2608 | ###################################################################################### |
|---|
| 2609 | ## |
|---|
| 2610 | elif ksect == u'requestlink': |
|---|
| 2611 | ## - uuid; mip; tab; objective; grid; gridreq; comment |
|---|
| 2612 | kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'ref', 'refNote', 'refid'] |
|---|
| 2613 | kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'refid'] |
|---|
| 2614 | ##kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt','opar','ref', 'refNote', 'refid'] |
|---|
| 2615 | |
|---|
| 2616 | for k,ll in self.rqLnks.items(): |
|---|
| 2617 | if k[0] != '_': |
|---|
| 2618 | assert len(ll) == len(kk), 'Bad record length in requestLink' |
|---|
| 2619 | thisl = string.strip(str( ll[2] ) ) |
|---|
| 2620 | uid = str( ll[0] ).strip() |
|---|
| 2621 | item = self.doc.createElement( 'item' ) |
|---|
| 2622 | for j in range(len(kk)): |
|---|
| 2623 | thiskk = kk[j] |
|---|
| 2624 | ## preset moved to request item .... so it can also be used on a per-experiment basis |
|---|
| 2625 | if thiskk != 'preset': |
|---|
| 2626 | if kk[j] == 'comment': |
|---|
| 2627 | v = utils_wb.uniCleanFunc( ll[j] ) |
|---|
| 2628 | else: |
|---|
| 2629 | v = str( ll[j] ) |
|---|
| 2630 | |
|---|
| 2631 | ### redirect to new variable group if specified in mappings |
|---|
| 2632 | ## |
|---|
| 2633 | item.setAttribute( thiskk, v ) |
|---|
| 2634 | |
|---|
| 2635 | item.setAttribute( 'label', thisl.replace( '.', '-' ) ) |
|---|
| 2636 | item.setAttribute( 'title', thisl ) |
|---|
| 2637 | self.ff[ksect][0].appendChild( item ) |
|---|
| 2638 | ################################## |
|---|
| 2639 | |
|---|
| 2640 | def _editRequestVar01( self, item, mm, t, thisd, vid=None ): |
|---|
| 2641 | if vid == None: |
|---|
| 2642 | vid = mm[0].vid2 |
|---|
| 2643 | item.setAttribute( 'vid', vid ) |
|---|
| 2644 | item.setAttribute( 'label', mm[0].lab2 ) |
|---|
| 2645 | thisd['vid'] = vid |
|---|
| 2646 | thisd['label'] = mm[0].lab2 |
|---|
| 2647 | print 'INFO.fffff.00001:', mm[0].lab2, mm[0].vid2 |
|---|
| 2648 | othd = None |
|---|
| 2649 | if len( mm ) > 1: |
|---|
| 2650 | othd = thisd.copy() |
|---|
| 2651 | assert len( mm ) < 3, 'Unable to deal with triple clone' |
|---|
| 2652 | cln = item.cloneNode(True) |
|---|
| 2653 | uid = str( uuid.uuid1() ) |
|---|
| 2654 | cln.setAttribute( 'uid', uid ) |
|---|
| 2655 | cln.setAttribute( 'label', mm[1].lab2 ) |
|---|
| 2656 | othd['uid'] = uid |
|---|
| 2657 | othd['label'] = mm[1].lab2 |
|---|
| 2658 | self.ff['requestvar'][0].appendChild( cln ) |
|---|
| 2659 | |
|---|
| 2660 | print 'INFO.046.0003: request variable split:', t[1] |
|---|
| 2661 | |
|---|
| 2662 | def finishExptGrp(self): |
|---|
| 2663 | """finish exptGroup section and create exptgrouplink section""" |
|---|
| 2664 | this = self.doc.getElementsByTagName('exptGroup')[0] |
|---|
| 2665 | ### kk = ['uid','label','tierMin','ntot'] |
|---|
| 2666 | for k in self.exptPntUid.keys(): |
|---|
| 2667 | u,typ = self.exptPntUid[k] |
|---|
| 2668 | ## if not isg: |
|---|
| 2669 | ##item = self.doc.createElement( 'item' ) |
|---|
| 2670 | ##item.setAttribute( uid, u ) |
|---|
| 2671 | ##this = self.doc.getElementsByTagName('exptGroupLink')[0] |
|---|
| 2672 | ##dil = this.getElementsByTagName('item') |
|---|
| 2673 | ##for d in dil: |
|---|
| 2674 | ##this.removeChild(d) |
|---|
| 2675 | ##this.appendChild( item ) |
|---|
| 2676 | |
|---|
| 2677 | def reviewExpt(self): |
|---|
| 2678 | """Prepare cross-linking information to populate requestitem, exptgroup and exptgrouplink""" |
|---|
| 2679 | sh = shelve.open( '../framework/sh_links/exptMappings', 'r' ) |
|---|
| 2680 | extraMappings={ 'LS3MIP-1, LS3MIP-2':['LmipH', 'LmipF'], \ |
|---|
| 2681 | 'LULCC_hist, CPL_1pctDF, LND_DF, ATM_DF, CPL_DF, CPL_noLULCC_hist_conc (LUMIP)':['LULCC_hist', 'CPL_1pctDF', 'LND_DF', 'ATM_DF', 'CPL_DF', 'CPL_noLULCC_hist_conc'] \ |
|---|
| 2682 | } |
|---|
| 2683 | |
|---|
| 2684 | ## |
|---|
| 2685 | ## extra fix for HighResMIP .... after encoding in template made consistent with other MIPs |
|---|
| 2686 | ## |
|---|
| 2687 | shm = {} |
|---|
| 2688 | for k in sh.keys(): |
|---|
| 2689 | if string.find( k, ' ' ) != -1 and k[:4] == 'High': |
|---|
| 2690 | k1 = string.split( k)[0] |
|---|
| 2691 | assert k1 not in shm, 'Duplicate abbreviated key: %s [%s and %s]' % (k1,k,shm[k1]) |
|---|
| 2692 | shm[k1] = k |
|---|
| 2693 | ks = sorted( self.exptPnt.keys() ) |
|---|
| 2694 | self.exptPntMap = collections.defaultdict( list ) |
|---|
| 2695 | self.exptPntUid = {} |
|---|
| 2696 | ner = 0 |
|---|
| 2697 | ngr = 0 |
|---|
| 2698 | nmr = 0 |
|---|
| 2699 | nuns = 0 |
|---|
| 2700 | nls = 0 |
|---|
| 2701 | for k0 in ks: |
|---|
| 2702 | k = string.replace( string.replace( k0, '\n', ' ' ), ' ', ' ') |
|---|
| 2703 | thisl = None |
|---|
| 2704 | if sh.has_key(k): |
|---|
| 2705 | thisl = sh[k] |
|---|
| 2706 | elif shm.has_key(k): |
|---|
| 2707 | thisl = sh[shm[k]] |
|---|
| 2708 | elif extraMappings.has_key(k): |
|---|
| 2709 | thisl = extraMappings[k] |
|---|
| 2710 | elif self.experiments.has_key(k): |
|---|
| 2711 | thisl = [k,] |
|---|
| 2712 | else: |
|---|
| 2713 | thisl = [k,] |
|---|
| 2714 | print 'WARN.080.00010: no mapping founds for experiment %s' % k |
|---|
| 2715 | |
|---|
| 2716 | |
|---|
| 2717 | if thisl != None: |
|---|
| 2718 | thisu = None |
|---|
| 2719 | thisls = set( thisl ) |
|---|
| 2720 | linkType = None |
|---|
| 2721 | for k2 in thisls: |
|---|
| 2722 | ok = True |
|---|
| 2723 | if self.experiments.has_key(k2): |
|---|
| 2724 | ner += 1 |
|---|
| 2725 | linkType = 'expt' |
|---|
| 2726 | thisu = self.experiments[k2] |
|---|
| 2727 | elif self.experimentGrp.has_key(k2): |
|---|
| 2728 | ngr += 1 |
|---|
| 2729 | thisu = self.experimentGrp[k2] |
|---|
| 2730 | linkType = 'grp' |
|---|
| 2731 | elif k2 in self.mips: |
|---|
| 2732 | nmr += 1 |
|---|
| 2733 | thisu = k2 |
|---|
| 2734 | linkType = 'mip' |
|---|
| 2735 | else: |
|---|
| 2736 | k2c = dreq_utils.labcoerce(k2) |
|---|
| 2737 | if self.experimentGrp.has_key(k2c): |
|---|
| 2738 | ngr += 1 |
|---|
| 2739 | thisu = self.experimentGrp[k2c] |
|---|
| 2740 | linkType = 'grp' |
|---|
| 2741 | else: |
|---|
| 2742 | ok = False |
|---|
| 2743 | print 'ERROR.080.00999: target experiment not found: %s [%s]' % (k2,k) |
|---|
| 2744 | if not ok: |
|---|
| 2745 | self.exptPntMap[k].append( k2 ) |
|---|
| 2746 | ## |
|---|
| 2747 | ## store uid of expt group to be pointed at. |
|---|
| 2748 | ## |
|---|
| 2749 | if len(thisls) == 1 and thisu != None: |
|---|
| 2750 | self.exptPntUid[k0] = (thisu, linkType, thisls) |
|---|
| 2751 | else: |
|---|
| 2752 | self.exptPntUid[k0] = (str( uuid.uuid1() ), 'list', tuple(thisls) ) |
|---|
| 2753 | print 'INFO.expt.00001: ',k0, thisl, thisls, self.exptPntUid[k0] |
|---|
| 2754 | nls += 1 |
|---|
| 2755 | else: |
|---|
| 2756 | nuns += 1 |
|---|
| 2757 | self.exptPntUid[k0] = (None,None) |
|---|
| 2758 | |
|---|
| 2759 | sh.close() |
|---|
| 2760 | print 'INFO.080.00010: Expt refs: %s, Expt group refs: %s, MIP expt set refs: %s, unset: %s, lists: %s' % (ner,ngr,nmr, nuns, nls) |
|---|
| 2761 | |
|---|
| 2762 | def reviewObj(self): |
|---|
| 2763 | ## review objectives |
|---|
| 2764 | ## |
|---|
| 2765 | ol = [] |
|---|
| 2766 | fnd1 = set() |
|---|
| 2767 | for m in self.objectives.keys(): |
|---|
| 2768 | not_used = [] |
|---|
| 2769 | used = set() |
|---|
| 2770 | not_found = [] |
|---|
| 2771 | found = [] |
|---|
| 2772 | alist = ['All','All?','all'] |
|---|
| 2773 | aaa = [self.objectiveLinks[m].a.has_key(x) for x in ['All','All?','all'] ] |
|---|
| 2774 | us1 = set() |
|---|
| 2775 | for k in self.objectiveLinks[m].a: |
|---|
| 2776 | x = None |
|---|
| 2777 | if k == 'AllExceptPaleorcms': |
|---|
| 2778 | x = self.objectives[m].keys() |
|---|
| 2779 | x.pop( x.index( 'Paleorcms' ) ) |
|---|
| 2780 | elif k == 'AllExceptNaturalVariability': |
|---|
| 2781 | x = self.objectives[m].keys() |
|---|
| 2782 | x.pop( x.index( 'NaturalVariability' ) ) |
|---|
| 2783 | |
|---|
| 2784 | if x != None: |
|---|
| 2785 | fnd1.add(k) |
|---|
| 2786 | for i in x: |
|---|
| 2787 | used.add(i) |
|---|
| 2788 | us1.add(i) |
|---|
| 2789 | for u in self.objectiveLinks[m].a[k]: |
|---|
| 2790 | ol.append( (m,self.objectives[m][i], u ) ) |
|---|
| 2791 | |
|---|
| 2792 | allused = (m != 'GMMIP') and any( aaa ) |
|---|
| 2793 | for o in self.objectives[m].keys(): |
|---|
| 2794 | if allused or self.objectiveLinks[m].a.has_key(o): |
|---|
| 2795 | used.add(o) |
|---|
| 2796 | for u in self.objectiveLinks[m].a[o]: |
|---|
| 2797 | ol.append( (m,self.objectives[m][o], u ) ) |
|---|
| 2798 | for k in alist: |
|---|
| 2799 | if self.objectiveLinks[m].a.has_key(k): |
|---|
| 2800 | for u in self.objectiveLinks[m].a[k]: |
|---|
| 2801 | ol.append( (m,self.objectives[m][o], u ) ) |
|---|
| 2802 | elif o not in us1: |
|---|
| 2803 | not_used.append(o) |
|---|
| 2804 | |
|---|
| 2805 | for o in self.objectiveLinks[m].a.keys(): |
|---|
| 2806 | if (o in {'All','All?'} and len(self.objectives[m].keys()) > 0 ) or self.objectives[m].has_key(o) or o in fnd1: |
|---|
| 2807 | found.append(o) |
|---|
| 2808 | else: |
|---|
| 2809 | not_found.append(o) |
|---|
| 2810 | if len(not_used) > 0 or len(not_found) > 0: |
|---|
| 2811 | print 'WARN.080.00002: %s: OK %s: unused: %s: not found:%s' % (m,str(used),str(not_used),str(not_found)) |
|---|
| 2812 | else: |
|---|
| 2813 | print 'INFO.080.00002: %s: all objectives matched' % m |
|---|
| 2814 | self.objList = ol |
|---|
| 2815 | |
|---|
| 2816 | |
|---|
| 2817 | def finish(self): |
|---|
| 2818 | ## varChoice section |
|---|
| 2819 | self.choices() |
|---|
| 2820 | ## |
|---|
| 2821 | self.reviewObj() |
|---|
| 2822 | ## |
|---|
| 2823 | return |
|---|
| 2824 | #d1 = {} |
|---|
| 2825 | #for i in self.rqvgList: |
|---|
| 2826 | #d1[i['uid']] = i |
|---|
| 2827 | #this = self.doc.getElementsByTagName('requestVarGroup')[0] |
|---|
| 2828 | #dil = this.getElementsByTagName('item') |
|---|
| 2829 | kkk = 0 |
|---|
| 2830 | kk0 = [] |
|---|
| 2831 | ss = {} |
|---|
| 2832 | #for item in dil: |
|---|
| 2833 | #uid = item.getAttribute( 'uid' ) |
|---|
| 2834 | #refn = item.getAttribute( 'refNote' ) |
|---|
| 2835 | #bits = string.split( refn, '_' ) |
|---|
| 2836 | #if len(bits) == 2: |
|---|
| 2837 | #if string.find(bits[0], '.' ) != -1: |
|---|
| 2838 | #bits[0] = string.split( bits[0], '.')[-1] |
|---|
| 2839 | #ss[ tuple( bits ) ] = uid |
|---|
| 2840 | #if uid not in d1: |
|---|
| 2841 | #kk0.append( 'Record for uid %s not found' % uid ) |
|---|
| 2842 | #else: |
|---|
| 2843 | #lab = item.getAttribute( 'label' ) |
|---|
| 2844 | #if d1[uid]['label'] == lab: |
|---|
| 2845 | #kkk += 1 |
|---|
| 2846 | #else: |
|---|
| 2847 | #kk0.append( 'Mismatch in label for %s' % uid ) |
|---|
| 2848 | #print 'INFO.rvg.00001: %s records checked OK; %s failed' % (kkk,len(kk0)) |
|---|
| 2849 | notNeeded = ( ('Oyr','3dtr'), ('Amon','2d'), ('cf3hr','grid')) |
|---|
| 2850 | ##nnf = 0 |
|---|
| 2851 | dt1 = {} |
|---|
| 2852 | for s in tableUtils.sgset: |
|---|
| 2853 | a,b = string.split( s, '_' ) |
|---|
| 2854 | if (a,b) not in notNeeded: |
|---|
| 2855 | if (a,b) in ss: |
|---|
| 2856 | ## |
|---|
| 2857 | ## create lookup --- dt1[<sub-group id>] = <requestVarGroup uid> |
|---|
| 2858 | ## |
|---|
| 2859 | dt1[s] = ss[(a,b)] |
|---|
| 2860 | else: |
|---|
| 2861 | print 'WARN.rvg.00002: %s %s not found' % (a,b) |
|---|
| 2862 | nnf += 1 |
|---|
| 2863 | |
|---|
| 2864 | assert nnf == 0, 'Not all sub-groups found' |
|---|
| 2865 | ### |
|---|
| 2866 | ### read through request vars ... and look to see how many refer to subgroups identified in dt1: |
|---|
| 2867 | ### how many of these are identified in tableUtils, how many in tableUtils are missing? |
|---|
| 2868 | ### |
|---|
| 2869 | dt1vs = {} |
|---|
| 2870 | for s in dt1: |
|---|
| 2871 | dt1vs[ dt1[s] ] = s |
|---|
| 2872 | |
|---|
| 2873 | ##print 'INFO.rvg.00009: dt1vs: %s' % str(dt1vs) |
|---|
| 2874 | this = self.doc.getElementsByTagName('requestVar')[0] |
|---|
| 2875 | dil = this.getElementsByTagName('item') |
|---|
| 2876 | nf0 = 0 |
|---|
| 2877 | nf1 = 0 |
|---|
| 2878 | nf2 = 0 |
|---|
| 2879 | ss0 = set() |
|---|
| 2880 | sn0 = collections.defaultdict( int ) |
|---|
| 2881 | sn1 = collections.defaultdict( int ) |
|---|
| 2882 | sn2 = collections.defaultdict( int ) |
|---|
| 2883 | for item in dil: |
|---|
| 2884 | vgid = item.getAttribute( 'vgid' ) |
|---|
| 2885 | if vgid in dt1vs: |
|---|
| 2886 | s = dt1vs[vgid] |
|---|
| 2887 | nf0 += 1 |
|---|
| 2888 | sn0[s] += 1 |
|---|
| 2889 | vid = item.getAttribute( 'vid' ) |
|---|
| 2890 | ss0.add( vid ) |
|---|
| 2891 | if vid in tableUtils.vdict: |
|---|
| 2892 | nf1 += 1 |
|---|
| 2893 | sn1[s] += 1 |
|---|
| 2894 | else: |
|---|
| 2895 | this.removeChild(item) |
|---|
| 2896 | |
|---|
| 2897 | for vid in tableUtils.vdict: |
|---|
| 2898 | if vid not in ss0: |
|---|
| 2899 | var,ttl,mip,tab,sg,priority = tableUtils.vdict[vid] |
|---|
| 2900 | a,b = string.split( sg, '_' ) |
|---|
| 2901 | if (a,b) not in notNeeded: |
|---|
| 2902 | nf2 += 1 |
|---|
| 2903 | sn2[ tableUtils.vdict[vid][4] ] += 1 |
|---|
| 2904 | assert sg in dt1, 'Cannot locate requestVarGroupd for %s: %s' % (sg,str(dt1.keys())) |
|---|
| 2905 | vgid = dt1[sg] |
|---|
| 2906 | item = self.doc.createElement( 'item' ) |
|---|
| 2907 | item.setAttribute( 'uid', str(uuid.uuid1()) ) |
|---|
| 2908 | item.setAttribute( 'vid', vid ) |
|---|
| 2909 | item.setAttribute( 'vgid', vgid ) |
|---|
| 2910 | item.setAttribute( 'title', '%s ((isd.009))' % var ) |
|---|
| 2911 | item.setAttribute( 'label', var ) |
|---|
| 2912 | item.setAttribute( 'mip', mip ) |
|---|
| 2913 | item.setAttribute( 'priority', priority ) |
|---|
| 2914 | this.appendChild( item ) |
|---|
| 2915 | |
|---|
| 2916 | print 'INFO.rvg.00010: variables found in sg rvgs: %s' % nf0 |
|---|
| 2917 | print 'INFO.rvg.00011: of these, number in tableUtils: %s' % nf1 |
|---|
| 2918 | print 'INFO.rvg.00012: variables in tableUtils not in sg rvgs: %s [from %s]' % (nf2,len(tableUtils.vdict.keys())) |
|---|
| 2919 | for k in sorted( sn0.keys() ): |
|---|
| 2920 | print 'INFO.rvg.00020: # in tableUtils [%s] %s [from %s]' % (k,sn0[k],sn1[k]) |
|---|
| 2921 | for k in sorted( sn2.keys() ): |
|---|
| 2922 | print 'INFO.rvg.00021: # in tableUtils [%s] not found %s' % (k,sn2[k]) |
|---|
| 2923 | |
|---|
| 2924 | |
|---|
| 2925 | def finish2(self): |
|---|
| 2926 | ## |
|---|
| 2927 | ## objectiveLink |
|---|
| 2928 | ## |
|---|
| 2929 | this = self.doc.getElementsByTagName('objectiveLink')[0] |
|---|
| 2930 | dil = this.getElementsByTagName('item') |
|---|
| 2931 | for d in dil: |
|---|
| 2932 | this.removeChild(d) |
|---|
| 2933 | for t in self.objList: |
|---|
| 2934 | item = self.doc.createElement( 'item' ) |
|---|
| 2935 | item.setAttribute( 'label', t[0] ) |
|---|
| 2936 | item.setAttribute( 'oid', t[1] ) |
|---|
| 2937 | item.setAttribute( 'rid', t[2] ) |
|---|
| 2938 | thisuuid = str( uuid.uuid1() ) |
|---|
| 2939 | item.setAttribute( 'uid', thisuuid ) |
|---|
| 2940 | this.appendChild( item ) |
|---|
| 2941 | ## |
|---|
| 2942 | ## remarks |
|---|
| 2943 | ## |
|---|
| 2944 | this = self.doc.getElementsByTagName('remarks')[0] |
|---|
| 2945 | dil = this.getElementsByTagName('item') |
|---|
| 2946 | for d in dil: |
|---|
| 2947 | this.removeChild(d) |
|---|
| 2948 | |
|---|
| 2949 | ##mn = self.doc.getElementsByTagName('main')[0] |
|---|
| 2950 | for item in self.doc.getElementsByTagName('item'): |
|---|
| 2951 | for k,v in item.attributes.items(): |
|---|
| 2952 | if type(v) not in ( type( u'x' ), type( 'x' ) ): |
|---|
| 2953 | print 'TYPE ERROR %s, %s: %s' % (item.parentNode.tagName, k,v) |
|---|
| 2954 | txt = self.doc.toprettyxml(indent='\t', newl='\n', encoding=None) |
|---|
| 2955 | if self.schemaMode == 'dreq2': |
|---|
| 2956 | ofn = 'trial2_20150831.xml' |
|---|
| 2957 | else: |
|---|
| 2958 | ofn = 'trial_20150831.xml' |
|---|
| 2959 | oo = open( ofn, 'w' ) |
|---|
| 2960 | lines = string.split( txt, '\n' ) |
|---|
| 2961 | for line in lines: |
|---|
| 2962 | l = utils_wb.uniCleanFunc( string.strip(line) ) |
|---|
| 2963 | if empty.match(l): |
|---|
| 2964 | continue |
|---|
| 2965 | else: |
|---|
| 2966 | oo.write(l + '\n') |
|---|
| 2967 | oo.close() |
|---|
| 2968 | |
|---|
| 2969 | def choices(self): |
|---|
| 2970 | """Populate the 'varChoice' section, using the information in the 'choices' dictionary in 'choicecfg'""" |
|---|
| 2971 | |
|---|
| 2972 | ccfg = choiceCfg.choices( self ) |
|---|
| 2973 | self.ccfg = ccfg |
|---|
| 2974 | ## |
|---|
| 2975 | ## choices |
|---|
| 2976 | ## |
|---|
| 2977 | this = self.doc.getElementsByTagName('varChoice')[0] |
|---|
| 2978 | dil = this.getElementsByTagName('item') |
|---|
| 2979 | for d in dil: |
|---|
| 2980 | this.removeChild(d) |
|---|
| 2981 | for k in ccfg.choices.keys(): |
|---|
| 2982 | item = self.doc.createElement( 'item' ) |
|---|
| 2983 | cc = ccfg.choices[k] |
|---|
| 2984 | for kk in cc.keys(): |
|---|
| 2985 | item.setAttribute( kk, cc[kk] ) |
|---|
| 2986 | this.appendChild( item ) |
|---|
| 2987 | ##print 'INFO.zzz.00008: Appending ',cc[kk] |
|---|
| 2988 | |
|---|
| 2989 | this = self.doc.getElementsByTagName('varChoiceLinkR')[0] |
|---|
| 2990 | dil = this.getElementsByTagName('item') |
|---|
| 2991 | for d in dil: |
|---|
| 2992 | this.removeChild(d) |
|---|
| 2993 | for cc in ccfg.varChoiceLnk: |
|---|
| 2994 | item = self.doc.createElement( 'item' ) |
|---|
| 2995 | ##print 'INFO.zzz', cc |
|---|
| 2996 | for i in range( len( ccfg.varChoiceLnkInfo ) ): |
|---|
| 2997 | item.setAttribute( ccfg.varChoiceLnkInfo[i], cc[i] ) |
|---|
| 2998 | this.appendChild( item ) |
|---|
| 2999 | |
|---|
| 3000 | this = self.doc.getElementsByTagName('varChoiceLinkC')[0] |
|---|
| 3001 | dil = this.getElementsByTagName('item') |
|---|
| 3002 | for d in dil: |
|---|
| 3003 | this.removeChild(d) |
|---|
| 3004 | for cc in ccfg.varChoiceLnkCfg: |
|---|
| 3005 | item = self.doc.createElement( 'item' ) |
|---|
| 3006 | for i in range( len( ccfg.varChoiceLnkCfgInfo ) ): |
|---|
| 3007 | item.setAttribute( ccfg.varChoiceLnkCfgInfo[i], cc[i] ) |
|---|
| 3008 | this.appendChild( item ) |
|---|
| 3009 | |
|---|
| 3010 | ##self.varChoiceLnkCfgInfo = [ 'uid', 'label', 'title', 'vid', 'cid', 'cfgid', 'cfg' |
|---|
| 3011 | |
|---|
| 3012 | def importRepl(self,rfile='uuidreplace.csv',mode='var'): |
|---|
| 3013 | for l in open(rfile).readlines(): |
|---|
| 3014 | bits = string.split( string.strip(l), '\t' ) |
|---|
| 3015 | assert len(bits) == 7, 'Could not parse item replacement file' |
|---|
| 3016 | if mode == 'var': |
|---|
| 3017 | self.repl[ bits[3] ].append( (bits[2],bits[0],bits[1] ) ) |
|---|
| 3018 | self.replItems[bits[0]] = bits[1] |
|---|
| 3019 | elif mode == 'cmv': |
|---|
| 3020 | self.rep['cmv'].v[ bits[3] ].append( (bits[2],bits[0],bits[1] ) ) |
|---|
| 3021 | self.rep['cmv'].items[bits[0]] = bits[1] |
|---|
| 3022 | |
|---|
| 3023 | |
|---|
| 3024 | def importRemove(self,rfile='uuidremove.csv'): |
|---|
| 3025 | for l in open(rfile).readlines(): |
|---|
| 3026 | self.remo[ string.strip(l) ] = 1 |
|---|
| 3027 | |
|---|
| 3028 | def importUpdate(self,rfile='uuidupdate.csv'): |
|---|
| 3029 | for l in open(rfile).readlines(): |
|---|
| 3030 | bits = string.split( string.strip(l), '\t' ) |
|---|
| 3031 | tgs = string.strip( bits[2] ) |
|---|
| 3032 | if string.find( tgs, ' ' ) == -1: |
|---|
| 3033 | tags = [tgs,] |
|---|
| 3034 | else: |
|---|
| 3035 | tags = string.split( tgs ) |
|---|
| 3036 | self.upda[ bits[0] ] = {'comment':bits[1], 'tags':tags, 'label':bits[3], 'title':bits[4] } |
|---|
| 3037 | |
|---|
| 3038 | def importInsert(self,rfile='uuidinsert.csv'): |
|---|
| 3039 | for l in open(rfile).readlines(): |
|---|
| 3040 | bits = string.split( string.strip(l), '\t' ) |
|---|
| 3041 | if bits[0] == 'unique': |
|---|
| 3042 | self.insert[bits[1]] = bits[3] |
|---|
| 3043 | |
|---|
| 3044 | mode = 'dreq2' |
|---|
| 3045 | sampleXml = '../framework/out/%sSample.xml' % mode |
|---|
| 3046 | from scansh02 import rq |
|---|
| 3047 | |
|---|
| 3048 | m = main(sampleXml, rq,run=True, schemaMode=mode, doRepl=False) |
|---|
| 3049 | |
|---|
