%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'month': = 'JOBID' almost month but not really,
# If you submit the job with job scheduler, above
#below are list of enviroment variable one can pass
#%env local='2"
# local : if True run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.
# ychunk=10 will group the original netcdf file to 10 by 10
# tchunk=1 will chunk the time coordinate one by one
#%env control=FWC_SSH
# name of control file to be used for computation/plots/save/
#%env file_exp=
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring this corresponds to file path path-XIOS.{month}/
#
#%env save= proceed saving? True or False , Default is setted as True
#%env plot= proceed plotting? True or False , Default is setted as True
#%env calc= proceed computation? or just load computed result? True or False , Default is setted as True
#%env save=False
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene5607.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False not local in tgcc rome local cluster starting This code is running on irene5607.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_ALPHA_MONITOR file experiment, read from ../lib/SEDNA_ALPHA_MONITOR.yaml on year= *3[0-9] on month= 21 outputpath= ../results/SEDNA_ALPHA_MONITOR/21/ daskreport= ../results/dask/2672798irene5607.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_21Ice_quantities/ CPU times: user 333 ms, sys: 236 ms, total: 568 ms Wall time: 10.1 s
Client
|
Cluster
|
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
Ice_quantities | param.e1te2t,icemod.sivelo,icemod.sivolu,icemo... | calc.Ice_quant(data) | ALL | Ice_intquant | None | (0,20) | cm s^(-1) | I-2 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read icemod ['sivolu', 'siconc', 'sivelo'] using load_data_xios reading icemod using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.21/SEDNA-ALPHA_1d_icemod_*3[0-9]_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set botpres: null deptht_bounds: null depthu_bounds: null iicestru: null iicestrv: null intstrx: null intstry: null mldkz5: null rhop_sig0: null siages: null sidive: null sisali: null sishea: null sistre: null sitemp: null snthic: null snvolu: null sometauy: null sozotaux: null time_centered_bounds: null time_counter_bounds: null utau_atmoce: null utau_iceoce: null uwspd10: null vtau_atmoce: null vtau_iceoce: null vwspd10: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ >
--------------------------------------------------------------------------- IndexError Traceback (most recent call last) <timed exec> in <module> /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in datas(catalog_url, dfi, month, year, daskreport, lazy) 601 # data=0 602 #else: --> 603 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 604 for s in paramdict: 605 print('param',s,'will be included in data') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in outputs(catalog_url, datadict, month, year, daskreport, lazy) 424 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"): 425 #ds=load_data_xios_patch(cat,filename,month,catalog_url) --> 426 ds = load_data_xios(cat,filename,items,month,year) if lazy else persist_data_xios(cat,filename,items,month,year) 427 extime=time.time() - start 428 print(' took', extime, 'seconds') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py in load_data_xios(cat, filename, items, month, year) 321 desc=cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).describe 322 print('using load_data_xios reading ',desc) --> 323 ds = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask() 324 ds = ds[items] 325 return ds ~/monitor/lib/python3.7/site-packages/intake_xarray/base.py in to_dask(self) 67 def to_dask(self): 68 """Return xarray object where variables are dask arrays""" ---> 69 return self.read_chunked() 70 71 def close(self): ~/monitor/lib/python3.7/site-packages/intake_xarray/base.py in read_chunked(self) 42 def read_chunked(self): 43 """Return xarray object (which will have chunks)""" ---> 44 self._load_metadata() 45 return self._ds 46 ~/monitor/lib/python3.7/site-packages/intake/source/base.py in _load_metadata(self) 234 """load metadata only if needed""" 235 if self._schema is None: --> 236 self._schema = self._get_schema() 237 self.dtype = self._schema.dtype 238 self.shape = self._schema.shape ~/monitor/lib/python3.7/site-packages/intake_xarray/base.py in _get_schema(self) 16 17 if self._ds is None: ---> 18 self._open_dataset() 19 20 metadata = { ~/monitor/lib/python3.7/site-packages/intake_xarray/netcdf.py in _open_dataset(self) 77 else: 78 _open_dataset = xr.open_dataset ---> 79 url = fsspec.open_local(url, **self.storage_options) 80 81 self._ds = _open_dataset(url, chunks=self.chunks, **kwargs) ~/monitor/lib/python3.7/site-packages/fsspec/core.py in open_local(url, mode, **storage_options) 457 raise ValueError("Can only ensure local files when reading") 458 of = open_files(url, mode=mode, **storage_options) --> 459 if not getattr(of[0].fs, "local_file", False): 460 raise ValueError( 461 "open_local can only be used on a filesystem which" IndexError: list index out of range
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) <timed eval> in <module> NameError: name 'data' is not defined