%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'month': = 'JOBID' almost month but not really,
# If you submit the job with job scheduler, above
#below are list of enviroment variable one can pass
#%env local='2"
# local : if True run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.
# ychunk=10 will group the original netcdf file to 10 by 10
# tchunk=1 will chunk the time coordinate one by one
#%env control=Ints_monitor
# name of control file to be used for computation/plots/save/
#%env file_exp=
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring this corresponds to file path path-XIOS.{month}/
#
#%env save= proceed saving? True or False , Default is setted as True
#%env plot= proceed plotting? True or False , Default is setted as True
#%env calc= proceed computation? or just load computed result? True or False , Default is setted as True
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene8004.c-irene.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False not local in tgcc c-irene.tgcc local FORCED tgcc local cluster starting This code is running on irene8004.c-irene.tgcc.ccc.cea.fr using SEDNA_ALPHA_YEAR file experiment, read from ../lib/SEDNA_ALPHA_YEAR.yaml on year= * on month= * outputpath= ../results/xlarge_SEDNA_ALPHA_YEAR/ daskreport= ../results/dask/7316892irene8004.c-irene.tgcc.ccc.cea.fr_SEDNA_ALPHA_YEAR_*compute/ CPU times: user 1.39 s, sys: 805 ms, total: 2.19 s Wall time: 35.1 s
Client
|
Cluster
|
df=load.controlfile(control)
#Take out 'later' tagged computations
df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
IceVolume | icemod.sivolu,param.e1te2t | (data.sivolu*data.e1te2t).sum(dim=('x','y'))/1... | ALL | ints | None | (0,40*10**3) | km^3 | I-2 | ||
IceArea | icemod.siconc,param.e1te2t | (data.siconc*data.e1te2t).sum(dim=('x','y'))/1... | ALL | ints | None | (0,20*10**6) | km^3 | I-2 | ||
IceExtent | icemod.siconc,param.e1te2t | (data.e1te2t).where(data.siconc>0.15).sum(dim=... | ALL | ints | None | (0,20*10**6) | km^3 | I-2 | ||
SSH_2D | gridT-2D.ssh,param.e1te2t | data.ssh.where(data.mask2d) | ALL | maps | Spectral_r | (-0.35,0.35) | m | M-2 | ||
IceThickness | icemod.sivolu | data.sivolu.where(data.sivolu >0) | ALL | maps | Spectral | (0,5) | m | M-4 | ||
IceConce | icemod.siconc | data.siconc.where(data.siconc >0) | ALL | maps | Blues | None | M-4 | |||
MLD_2D | gridT-2D.samldr10_1 | data.samldr10_1 | ALL | maps | Blues | (0,80) | m | M-5 | ||
AW_maxtemp_depth | gridT.votemper,gridS.vosaline,param.mask,param... | calc.AWTD4(data) | ALL | AWTD_map | jet | (0,800) | m | M-5 | ||
FWC_2D | gridS.vosaline,param.mask,param.e3t | calc.FWC2D_UFUNC(data) | BBFG | maps | Spectral_r | (0,24) | m | M-? | ||
Section | gridS.vosaline,gridT.votemper,gridV.vomecrty,p... | data.drop_vars('vozocrtx') | FramS | section | None | {'vosaline': (33,36.2), 'votemper': (-2,6), 'v... | None | S-1 | ||
Section | gridS.vosaline,gridT.votemper,gridU.vozocrtx,p... | data.drop_vars('vomecrty') | BFGS | section | None | {'vosaline': (28,35), 'votemper': (-2,2), 'voz... | None | S-2 | ||
votemper_Arc_B | gridT.votemper,param.depth,param.mask | data.votemper | Arc_B | mooring | rainbow | (-2.0,2.0) | C | x | ||
vosaline_Arc_B | gridT.vosalline,param.depth,param.mask | data.vosaline | Arc_B | mooring | rainbow | (28,34.4) | PSU | x |
Each computation consists of
%%time
#todo add 'year' here.
data=load.datas(catalog_url,df.Inputs,month,year,daskreport)
#print('#1 Data: created:')
#print('# if we raed too much file, we can do sel to take out some dates here')
data
../lib/SEDNA_ALPHA_YEAR.yaml using param_xios reading ../lib/SEDNA_ALPHA_YEAR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read gridS ['vosaline'] using load_data_xios reading gridS using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA/*/SEDNA-ALPHA-XIOS.*/SEDNA-ALPHA_1d_gridS_*_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 370.60780453681946 seconds 0 merging gridS ['vosaline'] 1 read gridT ['votemper', 'vosalline'] using load_data_xios reading gridT using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA/*/SEDNA-ALPHA-XIOS.*/SEDNA-ALPHA_1d_gridT_*_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ >
--------------------------------------------------------------------------- KeyError Traceback (most recent call last) ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _copy_listed(self, names) 1223 try: -> 1224 variables[name] = self._variables[name] 1225 except KeyError: KeyError: 'vosalline' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) <timed exec> in <module> /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in datas(catalog_url, dfi, month, year, daskreport, lazy) 589 para=param_xios(catalog_url) 590 datadict, paramdict = getdict(dfi) --> 591 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 592 for s in paramdict: 593 print('param',s,'will be included in data') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in outputs(catalog_url, datadict, month, year, daskreport, lazy) 416 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"): 417 #ds=load_data_xios_patch(cat,filename,month,catalog_url) --> 418 ds = load_data_xios(cat,filename,items,month,year) if lazy else persist_data_xios(cat,filename,items,month,year) 419 extime=time.time() - start 420 print(' took', extime, 'seconds') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in load_data_xios(cat, filename, items, month, year) 314 print('using load_data_xios reading ',desc) 315 ds = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask() --> 316 ds = ds[items] 317 return ds 318 ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in __getitem__(self, key) 1359 return self._construct_dataarray(key) 1360 else: -> 1361 return self._copy_listed(np.asarray(key)) 1362 1363 def __setitem__(self, key: Hashable, value) -> None: ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _copy_listed(self, names) 1225 except KeyError: 1226 ref_name, var_name, var = _get_virtual_variable( -> 1227 self._variables, name, self._level_coords, self.dims 1228 ) 1229 variables[var_name] = var ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _get_virtual_variable(variables, key, level_vars, dim_sizes) 169 ref_var = dim_var.to_index_variable().get_level_variable(ref_name) 170 else: --> 171 ref_var = variables[ref_name] 172 173 if var_name is None: KeyError: 'vosalline'
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) <timed eval> in <module> NameError: name 'data' is not defined