%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'catalog_url': we chose intake catalog for reading.
# 'month': = 'JOBID' almost month but not really,
# 'savefig': Do we save output in html? or not.
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to catalog name without path and .yaml
# If you submit the job with job scheduler, above
# 4 valus can be passed as enviroment variable.
#
#import os
#os.environ['file_exp']='SEDNA_ALPHA_MONITOR'
#os.environ['month']="22"
#os.environ['year']="*[3-4]" #"*0"
%%time
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
using host= irene4694.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True 10000000000 False not local in tgcc rome local cluster starting This code is running on irene4694.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_ALPHA_MONITOR file experiment, read from ../lib/SEDNA_ALPHA_MONITOR.yaml on year= * on month= 22 outputpath= ../results/rome_SEDNA_ALPHA_MONITOR/22/ daskreport= ../results/dask/irene4694.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_22/compute CPU times: user 317 ms, sys: 230 ms, total: 547 ms Wall time: 10.5 s
df=load.controlfile(control)
#
#df=df[~df.duplicated(subset='Plot' )]
df=df[df['Plot'] == 'maps']
df=df[3:4]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
AW_maxtemp_depth | gridT.votemper,gridS.vosaline,param.mask,param... | calc.AWD(data) | ALL | maps | jet | (0,800) | m | M-5 |
Each computation consists of
%%time
#todo add 'year' here.
data=load.datas(catalog_url,df.Inputs,month,year,daskreport)
print('#1 Data: created:')
print('# if we raed too much file, we can do sel to take out some datess..')
data
../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read gridS ['vosaline'] using load_data_xios reading gridS using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridS_*_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 167.52133226394653 seconds 0 merging gridS ['vosaline'] 1 read gridT ['votemper'] using load_data_xios reading gridT using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridT_*_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 73.58952593803406 seconds 1 merging gridT ['votemper'] took 0.3248164653778076 seconds param nav_lat will be included in data param nav_lon will be included in data param depth will be included in data param mask2d will be included in data param mask will be included in data #1 Data: created: # if we raed too much file, we can do sel to take out some datess.. CPU times: user 1min 37s, sys: 18.1 s, total: 1min 56s Wall time: 4min 16s
<xarray.Dataset> Dimensions: (t: 15, x: 6560, y: 6540, z: 150) Coordinates: * t (t) object 2004-06-16 12:00:00 ... 2004-06-30 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 143 144 145 146 147 148 149 150 nav_lat (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> depth (z, y, x) float64 dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(13, 6560), meta=np.ndarray> mask (z, y, x) bool dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> Data variables: vosaline (t, z, y, x) float32 dask.array<chunksize=(1, 150, 13, 6560), meta=np.ndarray> votemper (t, z, y, x) float32 dask.array<chunksize=(1, 150, 13, 6560), meta=np.ndarray>
array([cftime.DatetimeNoLeap(2004, 6, 16, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 19, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 20, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 22, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 23, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 24, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 25, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 27, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 28, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 29, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 30, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150])
|
|
|
|
|
|
|
#ds=data#.isel(t=slice(0,1))
ds=load.datas_rechunk(data,daskreport=daskreport,ysize=40)
sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12) start rechunking with (492, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 288) end of rechunk
#import os
#os.environ['save']='False'
#os.environ['calc']='True'
#os.environ['plot']='True'
%%time
monitor.auto(df,ds,savefig,daskreport,outputpath,file_exp='SEDNA'
)
switch:calcswitch,saveswitch,plotswitch True False True dtaa= calc.AWD(data) #3 Start computing count: <xarray.Dataset> Dimensions: () Data variables: vosaline int64 dask.array<chunksize=(), meta=np.ndarray> votemper int64 dask.array<chunksize=(), meta=np.ndarray>
<xarray.Dataset> Dimensions: () Data variables: vosaline int64 dask.array<chunksize=(), meta=np.ndarray> votemper int64 dask.array<chunksize=(), meta=np.ndarray>
|
|
nbytes: 5534514520 count: <xarray.Dataset> Dimensions: () Data variables: AWD int64 dask.array<chunksize=(), meta=np.ndarray>
<xarray.Dataset> Dimensions: (t: 15, x: 6560, y: 6540) Coordinates: * t (t) object 2004-06-16 12:00:00 ... 2004-06-30 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 nav_lat (y, x) float32 dask.array<chunksize=(492, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(492, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(492, 6560), meta=np.ndarray> Data variables: AWD (t, y, x) float64 dask.array<chunksize=(1, 492, 6560), meta=np.ndarray>
array([cftime.DatetimeNoLeap(2004, 6, 16, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 19, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 20, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 22, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 23, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 24, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 25, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 27, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 28, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 29, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 30, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
|
|
|
|
plots.maps(data,path=outputpath,filename=filename,save=savefig,cmap='jet',clim=(0,800),clabel='m') #5 Plotting SEDNA_maps_ALL_AW_maxtemp_depth ../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_maps_ALL_AW_maxtemp_depth_20040616-20040630.html starts plotting
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
0%| | 0/15 [00:00<?, ?it/s]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
7%|▋ | 1/15 [07:09<1:40:09, 429.28s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
13%|█▎ | 2/15 [13:58<1:30:30, 417.73s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
20%|██ | 3/15 [22:38<1:32:52, 464.35s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
27%|██▋ | 4/15 [29:25<1:20:57, 441.60s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 26, 12, 0, 0, 0)) distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('rechunk-merge-transpose-e71d3b77aa88470bff58e661a64e37a9', 10, 6, 0, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-e6da983b4b8e7d8bb703b99d70cdba60', 10, 0, 293, 0)"
Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/plotting/util.py", line 275, in get_plot_frame return map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/overlay.py", line 31, in dynamic_mul element = other[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1019, in dynamic_operation return apply(obj, *key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1011, in apply processed = self._process(element, key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 993, in _process return self.p.operation.process_element(element, key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 194, in process_element return self._apply(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 438, in _process op.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 219, in __call__ return element.apply(self, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 45, in pipelined_call result = __call__(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 204, in __call__ new_obj = apply_function(self._obj, **inner_kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 214, in __call__ return self._apply(element) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 40, in _process return element.map(self._process_element, self.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 237, in _process_element zs = element.dimension_values(2, flat=False) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1099, in dimension_values values = self.interface.values(self, dim, expanded, flat) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/xarray.py", line 385, in values data = data.compute() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 281, in compute (result,) = compute(self, traverse=False, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 563, in compute results = schedule(dsk, keys, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 2655, in get results = self.gather(packed, asynchronous=asynchronous, direct=direct) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1970, in gather asynchronous=asynchronous, File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 839, in sync self.loop, func, *args, callback_timeout=callback_timeout, **kwargs File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 340, in sync raise exc.with_traceback(tb) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 324, in f result[0] = yield future File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1829, in _gather raise exception.with_traceback(traceback) distributed.scheduler.KilledWorker: ("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 5, 0, 0)", <Worker 'tcp://127.0.0.1:45333', name: 12, memory: 0, processing: 10>) 33%|███▎ | 5/15 [34:07<1:03:59, 384.00s/it]
distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-c04aec06da96ba846878ce973f15f4f6', 0, 458, 0)" distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
40%|████ | 6/15 [41:31<1:00:38, 404.29s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
47%|████▋ | 7/15 [49:42<57:41, 432.65s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
53%|█████▎ | 8/15 [58:52<54:51, 470.20s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
60%|██████ | 9/15 [1:06:18<46:15, 462.55s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 21, 12, 0, 0, 0))
Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/plotting/util.py", line 275, in get_plot_frame return map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/overlay.py", line 31, in dynamic_mul element = other[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1019, in dynamic_operation return apply(obj, *key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1011, in apply processed = self._process(element, key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 993, in _process return self.p.operation.process_element(element, key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 194, in process_element return self._apply(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 438, in _process op.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 219, in __call__ return element.apply(self, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 45, in pipelined_call result = __call__(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 204, in __call__ new_obj = apply_function(self._obj, **inner_kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 214, in __call__ return self._apply(element) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 40, in _process return element.map(self._process_element, self.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 237, in _process_element zs = element.dimension_values(2, flat=False) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1099, in dimension_values values = self.interface.values(self, dim, expanded, flat) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/xarray.py", line 385, in values data = data.compute() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 281, in compute (result,) = compute(self, traverse=False, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 563, in compute results = schedule(dsk, keys, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 2655, in get results = self.gather(packed, asynchronous=asynchronous, direct=direct) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1970, in gather asynchronous=asynchronous, File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 839, in sync self.loop, func, *args, callback_timeout=callback_timeout, **kwargs File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 340, in sync raise exc.with_traceback(tb) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 324, in f result[0] = yield future File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1829, in _gather raise exception.with_traceback(traceback) distributed.scheduler.KilledWorker: ("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 8, 0, 0)", <Worker 'tcp://127.0.0.1:45324', name: 8, memory: 0, processing: 2>) 67%|██████▋ | 10/15 [1:10:29<33:05, 397.18s/it]
distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-0342a91fdd85385b86ab799ba0d3c19d', 5, 0, 0, 0)" distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.scheduler - ERROR - Couldn't gather keys {"('getitem-6a20bc08a9521f58c8379dbea66b4d50', 8, 0)": []} state: ['waiting'] workers: [] NoneType: None distributed.scheduler - ERROR - Workers don't have promised key: [], ('getitem-6a20bc08a9521f58c8379dbea66b4d50', 8, 0) NoneType: None distributed.client - WARNING - Couldn't gather 1 keys, rescheduling {"('getitem-6a20bc08a9521f58c8379dbea66b4d50', 8, 0)": ()}
73%|███████▎ | 11/15 [1:18:54<28:41, 430.31s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
80%|████████ | 12/15 [1:26:24<21:49, 436.39s/it]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 18, 12, 0, 0, 0)) distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting
Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/plotting/util.py", line 275, in get_plot_frame return map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/overlay.py", line 31, in dynamic_mul element = other[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1019, in dynamic_operation return apply(obj, *key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1011, in apply processed = self._process(element, key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 993, in _process return self.p.operation.process_element(element, key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 194, in process_element return self._apply(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 438, in _process op.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 219, in __call__ return element.apply(self, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 45, in pipelined_call result = __call__(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 204, in __call__ new_obj = apply_function(self._obj, **inner_kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 214, in __call__ return self._apply(element) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 40, in _process return element.map(self._process_element, self.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 237, in _process_element zs = element.dimension_values(2, flat=False) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1099, in dimension_values values = self.interface.values(self, dim, expanded, flat) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/xarray.py", line 385, in values data = data.compute() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 281, in compute (result,) = compute(self, traverse=False, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 563, in compute results = schedule(dsk, keys, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 2655, in get results = self.gather(packed, asynchronous=asynchronous, direct=direct) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1970, in gather asynchronous=asynchronous, File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 839, in sync self.loop, func, *args, callback_timeout=callback_timeout, **kwargs File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 340, in sync raise exc.with_traceback(tb) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 324, in f result[0] = yield future File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1829, in _gather raise exception.with_traceback(traceback) distributed.scheduler.KilledWorker: ("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 10, 0, 0)", <Worker 'tcp://127.0.0.1:40064', name: 6, memory: 0, processing: 3>) 87%|████████▋ | 13/15 [1:32:14<13:40, 410.08s/it]
distributed.nanny - WARNING - Restarting worker distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-c04aec06da96ba846878ce973f15f4f6', 0, 306, 0)" distributed.core - ERROR - Exception while handling op heartbeat_worker Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/core.py", line 496, in handle_comm result = handler(comm, **msg) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in heartbeat_worker parent._tasks[key]: duration for key, duration in executing.items() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/scheduler.py", line 3431, in <dictcomp> parent._tasks[key]: duration for key, duration in executing.items() KeyError: "('open_dataset-concatenate-c04aec06da96ba846878ce973f15f4f6', 0, 283, 0)" distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0), height=400, scale=1.0, width=400, x_range=(-3004627.5924886903, 3981641.8829163862), y_range=(-3694597.214786589, 3312147.721058284)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2004, 6, 17, 12, 0, 0, 0))
Traceback (most recent call last): File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/plotting/util.py", line 275, in get_plot_frame return map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 293, in dynamic_mul element = self[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/overlay.py", line 31, in dynamic_mul element = other[args] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1018, in dynamic_operation key, obj = resolve(key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1007, in resolve return key, map_obj[key] File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1331, in __getitem__ val = self._execute_callback(*tuple_key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 1100, in _execute_callback retval = self.callback(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/spaces.py", line 714, in __call__ ret = self.callable(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1019, in dynamic_operation return apply(obj, *key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 1011, in apply processed = self._process(element, key, kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/util/__init__.py", line 993, in _process return self.p.operation.process_element(element, key, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 194, in process_element return self._apply(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 438, in _process op.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 219, in __call__ return element.apply(self, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 45, in pipelined_call result = __call__(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/accessors.py", line 204, in __call__ new_obj = apply_function(self._obj, **inner_kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 214, in __call__ return self._apply(element) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/operation.py", line 141, in _apply ret = self._process(element, key) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 40, in _process return element.map(self._process_element, self.supported_types) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1216, in map return super(Dataset, self).map(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/dimension.py", line 709, in map return map_fn(self) if applies else self File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/geoviews/operation/projection.py", line 237, in _process_element zs = element.dimension_values(2, flat=False) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 201, in pipelined_fn result = method_fn(*args, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/__init__.py", line 1099, in dimension_values values = self.interface.values(self, dim, expanded, flat) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/holoviews/core/data/xarray.py", line 385, in values data = data.compute() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 281, in compute (result,) = compute(self, traverse=False, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/dask/base.py", line 563, in compute results = schedule(dsk, keys, **kwargs) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 2655, in get results = self.gather(packed, asynchronous=asynchronous, direct=direct) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1970, in gather asynchronous=asynchronous, File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 839, in sync self.loop, func, *args, callback_timeout=callback_timeout, **kwargs File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 340, in sync raise exc.with_traceback(tb) File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/utils.py", line 324, in f result[0] = yield future File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/tornado/gen.py", line 762, in run value = future.result() File "/ccc/cont003/home/ifremer/odakatin/monitor/lib/python3.7/site-packages/distributed/client.py", line 1829, in _gather raise exception.with_traceback(traceback) distributed.scheduler.KilledWorker: ("('rechunk-merge-transpose-d6290146f65b9c961e8b9a25c37791b6', 11, 0, 0)", <Worker 'tcp://127.0.0.1:36625', name: 5, memory: 0, processing: 3>) ../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_maps_ALL_AW_maxtemp_depth_20040616-20040630.html created
CPU times: user 52min 42s, sys: 4min 11s, total: 56min 53s Wall time: 1h 47min