%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'month': = 'JOBID' almost month but not really,
# If you submit the job with job scheduler, above
#below are list of enviroment variable one can pass
#%env local='2"
# local : if True run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.
# ychunk=10 will group the original netcdf file to 10 by 10
# tchunk=1 will chunk the time coordinate one by one
#%env control=FWC_SSH
# name of control file to be used for computation/plots/save/
#%env file_exp=
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring this corresponds to file path path-XIOS.{month}/
#
#%env save= proceed saving? True or False , Default is setted as True
#%env plot= proceed plotting? True or False , Default is setted as True
#%env calc= proceed computation? or just load computed result? True or False , Default is setted as True
#%env save=False
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene4468.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False not local in tgcc rome local cluster starting This code is running on irene4468.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_ALPHA_MONITOR file experiment, read from ../lib/SEDNA_ALPHA_MONITOR.yaml on year= * on month= 23 outputpath= ../results/SEDNA_ALPHA_MONITOR/23/ daskreport= ../results/dask/2529858irene4468.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_23IceClim/ CPU times: user 277 ms, sys: 231 ms, total: 509 ms Wall time: 9.71 s
Client
|
Cluster
|
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
IceClim | icemod.sivolu,icemod.siconc | data.chunk({ 't': -1 }).unify_chunks() | ALL | IceClim | Spectral | (0,5) | m | M-4 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read icemod ['sivolu', 'siconc'] using load_data_xios reading icemod using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.23/SEDNA-ALPHA_1d_icemod_*_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set botpres: null deptht_bounds: null depthu_bounds: null iicestru: null iicestrv: null intstrx: null intstry: null mldkz5: null rhop_sig0: null siages: null sidive: null sisali: null sishea: null sistre: null sitemp: null snthic: null snvolu: null sometauy: null sozotaux: null time_centered_bounds: null time_counter_bounds: null utau_atmoce: null utau_iceoce: null uwspd10: null vtau_atmoce: null vtau_iceoce: null vwspd10: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 448.8032035827637 seconds 0 merging icemod ['sivolu', 'siconc'] param nav_lon will be included in data param mask2d will be included in data param nav_lat will be included in data CPU times: user 3min 29s, sys: 35.6 s, total: 4min 4s Wall time: 7min 44s
<xarray.Dataset> Dimensions: (t: 31, x: 6560, y: 6540) Coordinates: nav_lat (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> * t (t) object 2004-07-01 12:00:00 ... 2004-07-31 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 mask2d (y, x) bool dask.array<chunksize=(13, 6560), meta=np.ndarray> Data variables: sivolu (t, y, x) float32 dask.array<chunksize=(1, 13, 6560), meta=np.ndarray> siconc (t, y, x) float32 dask.array<chunksize=(1, 13, 6560), meta=np.ndarray> Attributes: name: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RU... description: ice variables title: ice variables Conventions: CF-1.6 timeStamp: 2021-Aug-04 15:32:27 GMT uuid: be0d49a1-d456-4e73-a33e-e553bcdcaa0a ibegin: 0 ni: 6560 jbegin: 0 nj: 13 DOMAIN_number_total: 544 DOMAIN_number: 0 DOMAIN_dimensions_ids: [2 3] DOMAIN_size_global: [6560 6540] DOMAIN_size_local: [6560 13] DOMAIN_position_first: [1 1] DOMAIN_position_last: [6560 13] DOMAIN_halo_size_start: [0 0] DOMAIN_halo_size_end: [0 0] DOMAIN_type: box start_date: 20030101 output_frequency: 1d CONFIG: SEDNA CASE: ALPHA
|
|
array([cftime.DatetimeNoLeap(2004, 7, 1, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 2, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 3, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 4, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 5, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 6, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 7, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 8, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 9, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 10, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 11, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 12, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 13, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 14, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 15, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 16, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 17, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 18, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 19, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 20, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 21, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 22, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 23, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 24, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 25, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 26, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 27, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 28, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 29, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 30, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 31, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
|
|
|
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
#calc= True #save= True #plot= False Zone='ALL' Value='IceClim' cmap='Spectral' clabel='m' clim= (0, 5) outputpath='../results/SEDNA_ALPHA_MONITOR/23/' nc_outputpath='../nc_results/SEDNA_ALPHA_MONITOR/23/' filename='SEDNA_IceClim_ALL_IceClim' #3 Start computing dtaa= data.chunk({ 't': -1 }).unify_chunks()
<xarray.Dataset> Dimensions: (t: 31, x: 6560, y: 6540) Coordinates: nav_lat (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> * t (t) object 2004-07-01 12:00:00 ... 2004-07-31 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 mask2d (y, x) bool dask.array<chunksize=(13, 6560), meta=np.ndarray> Data variables: sivolu (t, y, x) float32 dask.array<chunksize=(31, 13, 6560), meta=np.ndarray> siconc (t, y, x) float32 dask.array<chunksize=(31, 13, 6560), meta=np.ndarray> Attributes: name: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RU... description: ice variables title: ice variables Conventions: CF-1.6 timeStamp: 2021-Aug-04 15:32:27 GMT uuid: be0d49a1-d456-4e73-a33e-e553bcdcaa0a ibegin: 0 ni: 6560 jbegin: 0 nj: 13 DOMAIN_number_total: 544 DOMAIN_number: 0 DOMAIN_dimensions_ids: [2 3] DOMAIN_size_global: [6560 6540] DOMAIN_size_local: [6560 13] DOMAIN_position_first: [1 1] DOMAIN_position_last: [6560 13] DOMAIN_halo_size_start: [0 0] DOMAIN_halo_size_end: [0 0] DOMAIN_type: box start_date: 20030101 output_frequency: 1d CONFIG: SEDNA CASE: ALPHA
|
|
array([cftime.DatetimeNoLeap(2004, 7, 1, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 2, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 3, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 4, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 5, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 6, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 7, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 8, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 9, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 10, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 11, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 12, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 13, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 14, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 15, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 16, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 17, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 18, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 19, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 20, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 21, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 22, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 23, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 24, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 25, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 26, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 27, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 28, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 29, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 30, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 7, 31, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
|
|
|
#4 Saving SEDNA_IceClim_ALL_IceClim dtaa=save.datas(data,plot=Plot,path=nc_outputpath,filename=filename) start saving data saving data in a file t (31,) 0 slice(0, 31, None)
distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 26, 262, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 13, 285, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 0, 172, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 5, 375, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 19, 268, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 6, 375, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 1, 172, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 4, 221, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 19, 262, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 24, 206, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 19, 339, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 13, 34, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-7aa2f95cf5427a8bf705cd466ccb6e45', 4, 66, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 25, 206, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 30, 206, 0)" distributed.nanny - WARNING - Restarting worker distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-ba9268119c3d5d45ffa60d45413dfbaa', 7, 344, 0)"
--------------------------------------------------------------------------- KilledWorker Traceback (most recent call last) <timed eval> in <module> /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py in auto(df, val, savefig, daskreport, outputpath, file_exp) 77 print('dtaa=save.datas(data,plot=Plot,path=nc_outputpath,filename=filename)' ) 78 with performance_report(filename=daskreport+"_save_"+step.Value+".html"): ---> 79 save.datas(data,plot=Plot,path=nc_outputpath,filename=filename) 80 # 5. Plot 81 if plotswitch=='True': /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py in datas(data, plot, path, filename) 22 twoD(data,path,filename,nested=False) 23 else : ---> 24 twoD(data,path,filename) 25 return None 26 /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py in twoD(data, path, filename, nested) 55 print('saving data in a file') 56 filesave=path+filename ---> 57 return to_mfnetcdf_map(data,prefix=filesave, nested=nested) 58 59 def twoD_onefile(data /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py in to_mfnetcdf_map(ds, prefix, nested) 226 ,template=template 227 ) --> 228 mapped.compute() 229 230 return mapped#.compute() ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in compute(self, **kwargs) 904 """ 905 new = self.copy(deep=False) --> 906 return new.load(**kwargs) 907 908 def _persist_inplace(self, **kwargs) -> "Dataset": ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in load(self, **kwargs) 739 740 # evaluate all the dask arrays simultaneously --> 741 evaluated_data = da.compute(*lazy_data.values(), **kwargs) 742 743 for k, data in zip(lazy_data, evaluated_data): ~/monitor/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs) 561 postcomputes.append(x.__dask_postcompute__()) 562 --> 563 results = schedule(dsk, keys, **kwargs) 564 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) 565 ~/monitor/lib/python3.7/site-packages/distributed/client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2653 should_rejoin = False 2654 try: -> 2655 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 2656 finally: 2657 for f in futures.values(): ~/monitor/lib/python3.7/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous) 1968 direct=direct, 1969 local_worker=local_worker, -> 1970 asynchronous=asynchronous, 1971 ) 1972 ~/monitor/lib/python3.7/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 837 else: 838 return sync( --> 839 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 840 ) 841 ~/monitor/lib/python3.7/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs) 338 if error[0]: 339 typ, exc, tb = error[0] --> 340 raise exc.with_traceback(tb) 341 else: 342 return result[0] ~/monitor/lib/python3.7/site-packages/distributed/utils.py in f() 322 if callback_timeout is not None: 323 future = asyncio.wait_for(future, callback_timeout) --> 324 result[0] = yield future 325 except Exception as exc: 326 error[0] = sys.exc_info() ~/monitor/lib/python3.7/site-packages/tornado/gen.py in run(self) 760 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info() ~/monitor/lib/python3.7/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker) 1827 exc = CancelledError(key) 1828 else: -> 1829 raise exception.with_traceback(traceback) 1830 raise exc 1831 if errors == "skip": KilledWorker: ("('create_eachfile-2923bc2c3aad110dcbd9a66ffd7e0eb4', 0, 0, 177)", <Worker 'tcp://127.0.0.1:36819', name: 13, memory: 0, processing: 8>)