%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'catalog_url': we chose intake catalog for reading.
# 'month': = 'JOBID' almost month but not really,
# 'savefig': Do we save output in html? or not.
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to catalog name without path and .yaml
# If you submit the job with job scheduler, above
# 4 valus can be passed as enviroment variable.
#
%%time
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
using host= irene6016.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True 10000000000 False not local in tgcc rome local cluster starting This code is running on irene6016.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_ALPHA_MONITOR file experiment, read from ../lib/SEDNA_ALPHA_MONITOR.yaml on year= *[8-9] on month= 22 outputpath= ../results/rome_SEDNA_ALPHA_MONITOR/22/ daskreport= ../results/dask/2452873irene6016.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_22compute/ CPU times: user 397 ms, sys: 240 ms, total: 637 ms Wall time: 12.7 s
Client
|
Cluster
|
df=load.controlfile(control)
#
#df=df[~df.duplicated(subset='Plot' )]
df=df[df['Plot'] == 'AWTD_map']
#df=df[3:4]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
AW_maxtemp_depth | gridT.votemper,gridS.vosaline,param.mask,param... | calc.AWTD4(data) | ALL | AWTD_map | jet | (0,800) | m | M-5 |
Each computation consists of
%%time
#todo add 'year' here.
data=load.datas(catalog_url,df.Inputs,month,year,daskreport)
print('#1 Data: created:')
print('# if we raed too much file, we can do sel to take out some datess..')
data
../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read gridS ['vosaline'] using load_data_xios reading gridS using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridS_*[8-9]_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 114.8224470615387 seconds 0 merging gridS ['vosaline'] 1 read gridT ['votemper'] using load_data_xios reading gridT using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridT_*[8-9]_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 19.304701566696167 seconds 1 merging gridT ['votemper'] took 0.2738478183746338 seconds param nav_lon will be included in data param mask2d will be included in data param nav_lat will be included in data param depth will be included in data param mask will be included in data #1 Data: created: # if we raed too much file, we can do sel to take out some datess.. CPU times: user 33.1 s, sys: 10.5 s, total: 43.6 s Wall time: 2min 31s
<xarray.Dataset> Dimensions: (t: 4, x: 6560, y: 6540, z: 150) Coordinates: * t (t) object 2004-06-18 12:00:00 ... 2004-06-29 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 143 144 145 146 147 148 149 150 nav_lon (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lat (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> depth (z, y, x) float64 dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> mask (z, y, x) bool dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> Data variables: vosaline (t, z, y, x) float32 dask.array<chunksize=(1, 150, 13, 6560), meta=np.ndarray> votemper (t, z, y, x) float32 dask.array<chunksize=(1, 150, 13, 6560), meta=np.ndarray>
array([cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 19, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 28, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 29, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150])
|
|
|
|
|
|
|
ds=load.datas_rechunk(data,daskreport=daskreport,ysize=40)
sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12) start rechunking with (492, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 288) end of rechunk
%%time
monitor.auto(df,ds,savefig,daskreport,outputpath,file_exp='SEDNA'
)
switch:calcswitch,saveswitch,plotswitch True False True dtaa= calc.AWTD4(data) #3 Start computing count: <xarray.Dataset> Dimensions: () Data variables: vosaline int64 dask.array<chunksize=(), meta=np.ndarray> votemper int64 dask.array<chunksize=(), meta=np.ndarray>
<xarray.Dataset> Dimensions: () Data variables: vosaline int64 dask.array<chunksize=(), meta=np.ndarray> votemper int64 dask.array<chunksize=(), meta=np.ndarray>
|
|
nbytes: 2445541632 count: <xarray.Dataset> Dimensions: () Data variables: AWT int64 dask.array<chunksize=(), meta=np.ndarray> AWD int64 dask.array<chunksize=(), meta=np.ndarray>
<xarray.Dataset> Dimensions: (t: 4, x: 6560, y: 6540) Coordinates: * t (t) object 2004-06-18 12:00:00 ... 2004-06-29 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 nav_lon (y, x) float32 dask.array<chunksize=(492, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(492, 6560), meta=np.ndarray> nav_lat (y, x) float32 dask.array<chunksize=(492, 6560), meta=np.ndarray> Data variables: AWT (t, y, x) float32 dask.array<chunksize=(1, 492, 6560), meta=np.ndarray> AWD (t, y, x) float64 dask.array<chunksize=(1, 492, 6560), meta=np.ndarray>
array([cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 19, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 28, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 29, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
|
|
|
|
|
plots.AWTD_map(data,path=outputpath,filename=filename,save=savefig,cmap='jet',clim=(0,800),clabel='m') #5 Plotting SEDNA_AWTD_map_ALL_AW_maxtemp_depth ../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_AWTD_map_ALL_AW_maxtemp_depth_20040618-20040629.html starts plotting
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker WARNING:param.dynamic_operation: Callable raised "KilledWorker("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=None, y_range=None) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=None, y_range=None) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 9, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7df73540e075bd0315aac077ff2bcde0', 0, 454, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7df73540e075bd0315aac077ff2bcde0', 0, 442, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 12, 0)" distributed.nanny - WARNING - Restarting worker
--------------------------------------------------------------------------- KilledWorker Traceback (most recent call last) <timed eval> in <module> /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py in auto(df, val, savefig, daskreport, outputpath, file_exp) 64 print(command, '#5 Plotting',filename ) 65 with performance_report(filename=daskreport+"_plot_"+step.Value+".html"): ---> 66 filename=eval(command ) 67 #if savefig: 68 print(filename,'created') /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py in <module> /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/plots.py in AWTD_map(data, path, filename, save, cmap, clim, clabel) 280 281 if save : --> 282 filename=save_html(plot,(path+filename),data.t) 283 #else: 284 # display(plot) /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/plots.py in save_html(html, filename, t) 55 filename=filename + tname +'.html' 56 print(filename,'starts plotting') ---> 57 hvplot.save(html, filename, resources=INLINE) 58 return filename 59 ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in save(obj, filename, fmt, backend, resources, toolbar, title, **kwargs) 818 filename = '.'.join(formats[:-1]) 819 return renderer_obj.save(obj, filename, fmt=fmt, resources=resources, --> 820 title=title) 821 822 ~/monitor/lib/python3.7/site-packages/holoviews/plotting/renderer.py in save(self_or_cls, obj, basename, fmt, key, info, options, resources, title, **kwargs) 622 if fmt in MIME_TYPES: 623 basename = '.'.join([basename, fmt]) --> 624 plot.layout.save(basename, embed=True, resources=resources, title=title) 625 return 626 ~/monitor/lib/python3.7/site-packages/panel/viewable.py in save(self, filename, title, resources, template, template_variables, embed, max_states, max_opts, embed_json, json_prefix, save_path, load_path, progress, embed_states) 722 template_variables, embed, max_states, max_opts, 723 embed_json, json_prefix, save_path, load_path, --> 724 progress, embed_states) 725 726 def server_doc(self, doc=None, title=None, location=True): ~/monitor/lib/python3.7/site-packages/panel/io/save.py in save(panel, filename, title, resources, template, template_variables, embed, max_states, max_opts, embed_json, json_prefix, save_path, load_path, progress, embed_states) 130 model = doc 131 else: --> 132 model = panel.get_root(doc, comm) 133 if embed: 134 embed_state( ~/monitor/lib/python3.7/site-packages/panel/viewable.py in get_root(self, doc, comm, preprocess) 480 """ 481 doc = init_doc(doc) --> 482 root = self._get_model(doc, comm=comm) 483 if preprocess: 484 self._preprocess(root) ~/monitor/lib/python3.7/site-packages/panel/layout/base.py in _get_model(self, doc, root, parent, comm) 110 if root is None: 111 root = model --> 112 objects = self._get_objects(model, [], doc, root, comm) 113 props = dict(self._init_properties(), objects=objects) 114 model.update(**self._process_param_change(props)) ~/monitor/lib/python3.7/site-packages/panel/layout/base.py in _get_objects(self, model, old_objects, doc, root, comm) 100 else: 101 try: --> 102 child = pane._get_model(doc, root, model, comm) 103 except RerenderError: 104 return self._get_objects(model, current_objects[:i], doc, root, comm) ~/monitor/lib/python3.7/site-packages/panel/pane/holoviews.py in _get_model(self, doc, root, parent, comm) 239 plot = self.object 240 else: --> 241 plot = self._render(doc, comm, root) 242 243 plot.pane = self ~/monitor/lib/python3.7/site-packages/panel/pane/holoviews.py in _render(self, doc, comm, root) 304 kwargs['comm'] = comm 305 --> 306 return renderer.get_plot(self.object, **kwargs) 307 308 def _cleanup(self, root): ~/monitor/lib/python3.7/site-packages/holoviews/plotting/bokeh/renderer.py in get_plot(self_or_cls, obj, doc, renderer, **kwargs) 71 combining the bokeh model with another plot. 72 """ ---> 73 plot = super(BokehRenderer, self_or_cls).get_plot(obj, doc, renderer, **kwargs) 74 if plot.document is None: 75 plot.document = Document() if self_or_cls.notebook_context else curdoc() ~/monitor/lib/python3.7/site-packages/holoviews/plotting/renderer.py in get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs) 218 219 # Initialize DynamicMaps with first data item --> 220 initialize_dynamic(obj) 221 222 if not renderer: ~/monitor/lib/python3.7/site-packages/holoviews/plotting/util.py in initialize_dynamic(obj) 250 continue 251 if not len(dmap): --> 252 dmap[dmap._initial_key()] 253 254 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __getitem__(self, key) 1329 # Not a cross product and nothing cached so compute element. 1330 if cache is not None: return cache -> 1331 val = self._execute_callback(*tuple_key) 1332 if data_slice: 1333 val = self._dataslice(val, data_slice) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args) 1098 1099 with dynamicmap_memoization(self.callback, self.streams): -> 1100 retval = self.callback(*args, **kwargs) 1101 return self._style(retval) 1102 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __call__(self, *args, **kwargs) 712 713 try: --> 714 ret = self.callable(*args, **kwargs) 715 except KeyError: 716 # KeyError is caught separately because it is used to signal ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in dynamic_mul(*args, **kwargs) 291 if isinstance(self, DynamicMap): 292 def dynamic_mul(*args, **kwargs): --> 293 element = self[args] 294 if reverse: 295 return other * element ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __getitem__(self, key) 1329 # Not a cross product and nothing cached so compute element. 1330 if cache is not None: return cache -> 1331 val = self._execute_callback(*tuple_key) 1332 if data_slice: 1333 val = self._dataslice(val, data_slice) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args) 1098 1099 with dynamicmap_memoization(self.callback, self.streams): -> 1100 retval = self.callback(*args, **kwargs) 1101 return self._style(retval) 1102 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __call__(self, *args, **kwargs) 712 713 try: --> 714 ret = self.callable(*args, **kwargs) 715 except KeyError: 716 # KeyError is caught separately because it is used to signal ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in dynamic_mul(*args, **kwargs) 291 if isinstance(self, DynamicMap): 292 def dynamic_mul(*args, **kwargs): --> 293 element = self[args] 294 if reverse: 295 return other * element ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __getitem__(self, key) 1329 # Not a cross product and nothing cached so compute element. 1330 if cache is not None: return cache -> 1331 val = self._execute_callback(*tuple_key) 1332 if data_slice: 1333 val = self._dataslice(val, data_slice) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args) 1098 1099 with dynamicmap_memoization(self.callback, self.streams): -> 1100 retval = self.callback(*args, **kwargs) 1101 return self._style(retval) 1102 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __call__(self, *args, **kwargs) 712 713 try: --> 714 ret = self.callable(*args, **kwargs) 715 except KeyError: 716 # KeyError is caught separately because it is used to signal ~/monitor/lib/python3.7/site-packages/holoviews/core/overlay.py in dynamic_mul(*args, **kwargs) 29 from .spaces import Callable 30 def dynamic_mul(*args, **kwargs): ---> 31 element = other[args] 32 return self * element 33 callback = Callable(dynamic_mul, inputs=[self, other]) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __getitem__(self, key) 1329 # Not a cross product and nothing cached so compute element. 1330 if cache is not None: return cache -> 1331 val = self._execute_callback(*tuple_key) 1332 if data_slice: 1333 val = self._dataslice(val, data_slice) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args) 1098 1099 with dynamicmap_memoization(self.callback, self.streams): -> 1100 retval = self.callback(*args, **kwargs) 1101 return self._style(retval) 1102 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __call__(self, *args, **kwargs) 712 713 try: --> 714 ret = self.callable(*args, **kwargs) 715 except KeyError: 716 # KeyError is caught separately because it is used to signal ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in dynamic_operation(*key, **kwargs) 1016 1017 def dynamic_operation(*key, **kwargs): -> 1018 key, obj = resolve(key, kwargs) 1019 return apply(obj, *key, **kwargs) 1020 ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in resolve(key, kwargs) 1005 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key: 1006 key = tuple(kwargs[k] for k in map_obj._posarg_keys) -> 1007 return key, map_obj[key] 1008 1009 def apply(element, *key, **kwargs): ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __getitem__(self, key) 1329 # Not a cross product and nothing cached so compute element. 1330 if cache is not None: return cache -> 1331 val = self._execute_callback(*tuple_key) 1332 if data_slice: 1333 val = self._dataslice(val, data_slice) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args) 1098 1099 with dynamicmap_memoization(self.callback, self.streams): -> 1100 retval = self.callback(*args, **kwargs) 1101 return self._style(retval) 1102 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __call__(self, *args, **kwargs) 712 713 try: --> 714 ret = self.callable(*args, **kwargs) 715 except KeyError: 716 # KeyError is caught separately because it is used to signal ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in dynamic_operation(*key, **kwargs) 1016 1017 def dynamic_operation(*key, **kwargs): -> 1018 key, obj = resolve(key, kwargs) 1019 return apply(obj, *key, **kwargs) 1020 ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in resolve(key, kwargs) 1005 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key: 1006 key = tuple(kwargs[k] for k in map_obj._posarg_keys) -> 1007 return key, map_obj[key] 1008 1009 def apply(element, *key, **kwargs): ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __getitem__(self, key) 1329 # Not a cross product and nothing cached so compute element. 1330 if cache is not None: return cache -> 1331 val = self._execute_callback(*tuple_key) 1332 if data_slice: 1333 val = self._dataslice(val, data_slice) ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args) 1098 1099 with dynamicmap_memoization(self.callback, self.streams): -> 1100 retval = self.callback(*args, **kwargs) 1101 return self._style(retval) 1102 ~/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py in __call__(self, *args, **kwargs) 712 713 try: --> 714 ret = self.callable(*args, **kwargs) 715 except KeyError: 716 # KeyError is caught separately because it is used to signal ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in dynamic_operation(*key, **kwargs) 1017 def dynamic_operation(*key, **kwargs): 1018 key, obj = resolve(key, kwargs) -> 1019 return apply(obj, *key, **kwargs) 1020 1021 operation = self.p.operation ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in apply(element, *key, **kwargs) 1009 def apply(element, *key, **kwargs): 1010 kwargs = dict(util.resolve_dependent_kwargs(self.p.kwargs), **kwargs) -> 1011 processed = self._process(element, key, kwargs) 1012 if (self.p.link_dataset and isinstance(element, Dataset) and 1013 isinstance(processed, Dataset) and processed._dataset is None): ~/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py in _process(self, element, key, kwargs) 991 elif isinstance(self.p.operation, Operation): 992 kwargs = {k: v for k, v in kwargs.items() if k in self.p.operation.param} --> 993 return self.p.operation.process_element(element, key, **kwargs) 994 else: 995 return self.p.operation(element, **kwargs) ~/monitor/lib/python3.7/site-packages/holoviews/core/operation.py in process_element(self, element, key, **params) 192 self.p = param.ParamOverrides(self, params, 193 allow_extra_keywords=self._allow_extra_keywords) --> 194 return self._apply(element, key) 195 196 ~/monitor/lib/python3.7/site-packages/holoviews/core/operation.py in _apply(self, element, key) 139 if not in_method: 140 element._in_method = True --> 141 ret = self._process(element, key) 142 if hasattr(element, '_in_method') and not in_method: 143 element._in_method = in_method ~/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py in _process(self, element, key) 436 for op in self._operations: 437 element = element.map(op.instance(projection=self.p.projection), --> 438 op.supported_types) 439 return element ~/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py in pipelined_fn(*args, **kwargs) 199 200 try: --> 201 result = method_fn(*args, **kwargs) 202 if PipelineMeta.disable: 203 return result ~/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py in map(self, *args, **kwargs) 1214 1215 def map(self, *args, **kwargs): -> 1216 return super(Dataset, self).map(*args, **kwargs) 1217 map.__doc__ = LabelledData.map.__doc__ 1218 ~/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py in map(self, map_fn, specs, clone) 707 return deep_mapped 708 else: --> 709 return map_fn(self) if applies else self 710 711 ~/monitor/lib/python3.7/site-packages/holoviews/core/operation.py in __call__(self, element, **kwargs) 217 kwargs['per_element'] = self._per_element 218 kwargs['link_dataset'] = self._propagate_dataset --> 219 return element.apply(self, **kwargs) 220 221 ~/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py in pipelined_call(*args, **kwargs) 43 44 try: ---> 45 result = __call__(*args, **kwargs) 46 47 if not in_method: ~/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py in __call__(self, apply_function, streams, link_inputs, link_dataset, dynamic, per_element, **kwargs) 202 if hasattr(apply_function, 'dynamic'): 203 inner_kwargs['dynamic'] = False --> 204 new_obj = apply_function(self._obj, **inner_kwargs) 205 if (link_dataset and isinstance(self._obj, Dataset) and 206 isinstance(new_obj, Dataset) and new_obj._dataset is None): ~/monitor/lib/python3.7/site-packages/holoviews/core/operation.py in __call__(self, element, **kwargs) 212 elif ((self._per_element and isinstance(element, Element)) or 213 (not self._per_element and isinstance(element, ViewableElement))): --> 214 return self._apply(element) 215 elif 'streams' not in kwargs: 216 kwargs['streams'] = self.p.streams ~/monitor/lib/python3.7/site-packages/holoviews/core/operation.py in _apply(self, element, key) 139 if not in_method: 140 element._in_method = True --> 141 ret = self._process(element, key) 142 if hasattr(element, '_in_method') and not in_method: 143 element._in_method = in_method ~/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py in _process(self, element, key) 38 39 def _process(self, element, key=None): ---> 40 return element.map(self._process_element, self.supported_types) 41 42 ~/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py in pipelined_fn(*args, **kwargs) 199 200 try: --> 201 result = method_fn(*args, **kwargs) 202 if PipelineMeta.disable: 203 return result ~/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py in map(self, *args, **kwargs) 1214 1215 def map(self, *args, **kwargs): -> 1216 return super(Dataset, self).map(*args, **kwargs) 1217 map.__doc__ = LabelledData.map.__doc__ 1218 ~/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py in map(self, map_fn, specs, clone) 707 return deep_mapped 708 else: --> 709 return map_fn(self) if applies else self 710 711 ~/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py in _process_element(self, element) 235 for kd in element.kdims) 236 --> 237 zs = element.dimension_values(2, flat=False) 238 if irregular: 239 X, Y = [np.asarray(element.interface.coords( ~/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py in pipelined_fn(*args, **kwargs) 199 200 try: --> 201 result = method_fn(*args, **kwargs) 202 if PipelineMeta.disable: 203 return result ~/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py in dimension_values(self, dimension, expanded, flat) 1097 """ 1098 dim = self.get_dimension(dimension, strict=True) -> 1099 values = self.interface.values(self, dim, expanded, flat) 1100 if dim.nodata is not None: 1101 # Ensure nodata applies to boolean data in py2 ~/monitor/lib/python3.7/site-packages/holoviews/core/data/xarray.py in values(cls, dataset, dim, expanded, flat, compute, keep_index) 383 da = dask_array_module() 384 if compute and da and isinstance(data, da.Array): --> 385 data = data.compute() 386 if is_cupy(data): 387 import cupy ~/monitor/lib/python3.7/site-packages/dask/base.py in compute(self, **kwargs) 279 dask.base.compute 280 """ --> 281 (result,) = compute(self, traverse=False, **kwargs) 282 return result 283 ~/monitor/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs) 561 postcomputes.append(x.__dask_postcompute__()) 562 --> 563 results = schedule(dsk, keys, **kwargs) 564 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) 565 ~/monitor/lib/python3.7/site-packages/distributed/client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2653 should_rejoin = False 2654 try: -> 2655 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 2656 finally: 2657 for f in futures.values(): ~/monitor/lib/python3.7/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous) 1968 direct=direct, 1969 local_worker=local_worker, -> 1970 asynchronous=asynchronous, 1971 ) 1972 ~/monitor/lib/python3.7/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 837 else: 838 return sync( --> 839 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 840 ) 841 ~/monitor/lib/python3.7/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs) 338 if error[0]: 339 typ, exc, tb = error[0] --> 340 raise exc.with_traceback(tb) 341 else: 342 return result[0] ~/monitor/lib/python3.7/site-packages/distributed/utils.py in f() 322 if callback_timeout is not None: 323 future = asyncio.wait_for(future, callback_timeout) --> 324 result[0] = yield future 325 except Exception as exc: 326 error[0] = sys.exc_info() ~/monitor/lib/python3.7/site-packages/tornado/gen.py in run(self) 760 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info() ~/monitor/lib/python3.7/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker) 1827 exc = CancelledError(key) 1828 else: -> 1829 raise exception.with_traceback(traceback) 1830 raise exc 1831 if errors == "skip": KilledWorker: ("('broadcast_to-sum-sum-aggregate-where-getitem-945a51d3b3a367de7a88846641654efa', 10, 0)", <Worker 'tcp://127.0.0.1:40769', name: 6, memory: 0, processing: 2>)