%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'month': = 'JOBID' almost month but not really,
# If you submit the job with job scheduler, above
#below are list of enviroment variable one can pass
#%env local='2"
# local : if True run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.
# ychunk=10 will group the original netcdf file to 10 by 10
# tchunk=1 will chunk the time coordinate one by one
#%env control=Ints_monitor
# name of control file to be used for computation/plots/save/
#%env file_exp=
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring this corresponds to file path path-XIOS.{month}/
#
#%env save= proceed saving? True or False , Default is setted as True
#%env plot= proceed plotting? True or False , Default is setted as True
#%env calc= proceed computation? or just load computed result? True or False , Default is setted as True
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene6009.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False not local in tgcc rome local cluster starting This code is running on irene6009.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_ALPHA_MONITOR file experiment, read from ../lib/SEDNA_ALPHA_MONITOR.yaml on year= * on month= 22 outputpath= ../results/rome_SEDNA_ALPHA_MONITOR/22/ daskreport= ../results/dask/2474302irene6009.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_22vosaline_Arc_B_moni/ CPU times: user 420 ms, sys: 257 ms, total: 677 ms Wall time: 12.5 s
Client
|
Cluster
|
df=load.controlfile(control)
#Take out 'later' tagged computations
df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
vosaline_Arc_B | gridT.vosalline,param.depth,param.mask | data.vosaline | Arc_B | mooring | rainbow | (28,34.4) | PSU | x |
Each computation consists of
%%time
#todo add 'year' here.
data=load.datas(catalog_url,df.Inputs,month,year,daskreport)
#print('#1 Data: created:')
#print('# if we raed too much file, we can do sel to take out some dates here')
data
../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read gridT ['vosalline'] using load_data_xios reading gridT using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridT_*_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ >
--------------------------------------------------------------------------- KeyError Traceback (most recent call last) ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _copy_listed(self, names) 1223 try: -> 1224 variables[name] = self._variables[name] 1225 except KeyError: KeyError: 'vosalline' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) <timed exec> in <module> /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in datas(catalog_url, dfi, month, year, daskreport, lazy) 589 para=param_xios(catalog_url) 590 datadict, paramdict = getdict(dfi) --> 591 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 592 for s in paramdict: 593 print('param',s,'will be included in data') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in outputs(catalog_url, datadict, month, year, daskreport, lazy) 416 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"): 417 #ds=load_data_xios_patch(cat,filename,month,catalog_url) --> 418 ds = load_data_xios(cat,filename,items,month,year) if lazy else persist_data_xios(cat,filename,items,month,year) 419 extime=time.time() - start 420 print(' took', extime, 'seconds') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in load_data_xios(cat, filename, items, month, year) 314 print('using load_data_xios reading ',desc) 315 ds = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask() --> 316 ds = ds[items] 317 return ds 318 ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in __getitem__(self, key) 1359 return self._construct_dataarray(key) 1360 else: -> 1361 return self._copy_listed(np.asarray(key)) 1362 1363 def __setitem__(self, key: Hashable, value) -> None: ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _copy_listed(self, names) 1225 except KeyError: 1226 ref_name, var_name, var = _get_virtual_variable( -> 1227 self._variables, name, self._level_coords, self.dims 1228 ) 1229 variables[var_name] = var ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _get_virtual_variable(variables, key, level_vars, dim_sizes) 169 ref_var = dim_var.to_index_variable().get_level_variable(ref_name) 170 else: --> 171 ref_var = variables[ref_name] 172 173 if var_name is None: KeyError: 'vosalline'
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) <timed eval> in <module> NameError: name 'data' is not defined