%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
If you submit the job with job scheduler; below are list of enviroment variable one can pass
local : if True run dask local cluster, if not true, put number of workers setted in the 'local' if no 'local ' given, local will be setted automatically to 'True'
%env ychunk='2', #%env tchunk='2'
controls chunk. 'False' sets no modification from original netcdf file's chunk.
ychunk=10 will group the original netcdf file to 10 by 10
tchunk=1 will chunk the time coordinate one by one
%env file_exp=
'file_exp': Which 'experiment' name is it? this corresopnds to intake catalog name without path and .yaml
#%env year=
for Validation, this correspoinds to path/year/month 's year for monitoring, this corresponids to 'date' having means do all files in the monitoring directory setting it as 0[0-9] &1[0-9]& [2-3][0-9], the job can be separated in three lots. For DELTA experiment, year corresponds to really 'year'
%env month=
for monitoring this corresponds to file path path-XIOS.{month}/
For DELTA experiment, year corresponds to really 'month'
proceed saving? True or False , Default is setted as True
proceed plotting? True or False , Default is setted as True
proceed computation? or just load computed result? True or False , Default is setted as True
save output file used for plotting
using kerchunked file -> False, not using kerhcunk -> True
name of control file to be used for computation/plots/save/ We have number of M_xxx.csv
Monitor.sh calls M_MLD_2D
and AWTD.sh, Fluxnet.sh, Siconc.sh, IceClim.sh, FWC_SSH.sh, Integrals.sh , Sections.sh
M_AWTMD
M_Fluxnet
M_Ice_quantities
M_IceClim M_IceConce M_IceThick
M_FWC_2D M_FWC_integrals M_FWC_SSH M_SSH_anomaly
M_Mean_temp_velo M_Mooring
M_Sectionx M_Sectiony
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local 20 local false ,workers 20 using host= irene4872.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= False workers 20 10000000000 mg1: rome dask jobqueue rome, slurmcluster starting This code is running on irene4872.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_DELTA_MONITOR file experiment, read from ../lib/SEDNA_DELTA_MONITOR.yaml on year= 2012 on month= 0[3-4] outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6462661irene4872.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_0[3-4]M_MLD_2D/ CPU times: user 679 ms, sys: 178 ms, total: 857 ms Wall time: 34.6 s
Client-82497de3-180a-11ed-8048-080038b93e37
Connection method: Cluster object | Cluster type: dask_jobqueue.SLURMCluster |
Dashboard: http://10.131.5.191:8787/status |
bf459de8
Dashboard: http://10.131.5.191:8787/status | Workers: 28 |
Total threads: 252 | Total memory: 428.40 GiB |
Scheduler-1a84a604-ae38-40c6-a3e5-1e166c60fcfb
Comm: tcp://10.131.5.191:37290 | Workers: 28 |
Dashboard: http://10.131.5.191:8787/status | Total threads: 252 |
Started: Just now | Total memory: 428.40 GiB |
Comm: tcp://10.131.13.176:44136 | Total threads: 9 |
Dashboard: http://10.131.13.176:46536/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:41770 | |
Local directory: /tmp/dask-worker-space/worker-0zjkl_ei |
Comm: tcp://10.131.13.176:42696 | Total threads: 9 |
Dashboard: http://10.131.13.176:43270/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:36582 | |
Local directory: /tmp/dask-worker-space/worker-ay1n_ccn |
Comm: tcp://10.131.13.176:35083 | Total threads: 9 |
Dashboard: http://10.131.13.176:46587/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:38303 | |
Local directory: /tmp/dask-worker-space/worker-dy8jq48k |
Comm: tcp://10.131.13.176:36225 | Total threads: 9 |
Dashboard: http://10.131.13.176:37895/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:38629 | |
Local directory: /tmp/dask-worker-space/worker-8e3n227o |
Comm: tcp://10.131.13.176:45517 | Total threads: 9 |
Dashboard: http://10.131.13.176:38923/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:46399 | |
Local directory: /tmp/dask-worker-space/worker-twlg2or0 |
Comm: tcp://10.131.13.176:45818 | Total threads: 9 |
Dashboard: http://10.131.13.176:40358/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:39745 | |
Local directory: /tmp/dask-worker-space/worker-my5yp5p2 |
Comm: tcp://10.131.13.176:43907 | Total threads: 9 |
Dashboard: http://10.131.13.176:38856/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:38709 | |
Local directory: /tmp/dask-worker-space/worker-tf2n1exn |
Comm: tcp://10.131.13.176:37999 | Total threads: 9 |
Dashboard: http://10.131.13.176:43556/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:45760 | |
Local directory: /tmp/dask-worker-space/worker-kw02t_6p |
Comm: tcp://10.131.13.176:37471 | Total threads: 9 |
Dashboard: http://10.131.13.176:43580/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:39366 | |
Local directory: /tmp/dask-worker-space/worker-nz73xocr |
Comm: tcp://10.131.13.176:36243 | Total threads: 9 |
Dashboard: http://10.131.13.176:40846/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:43610 | |
Local directory: /tmp/dask-worker-space/worker-302wd27r |
Comm: tcp://10.131.13.176:35224 | Total threads: 9 |
Dashboard: http://10.131.13.176:36550/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:34554 | |
Local directory: /tmp/dask-worker-space/worker-aqsjztvf |
Comm: tcp://10.131.13.176:35735 | Total threads: 9 |
Dashboard: http://10.131.13.176:33661/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:41475 | |
Local directory: /tmp/dask-worker-space/worker-qjvf0grx |
Comm: tcp://10.131.13.176:40577 | Total threads: 9 |
Dashboard: http://10.131.13.176:45938/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:40034 | |
Local directory: /tmp/dask-worker-space/worker-_xqkkwxn |
Comm: tcp://10.131.13.176:42059 | Total threads: 9 |
Dashboard: http://10.131.13.176:38269/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.13.176:35232 | |
Local directory: /tmp/dask-worker-space/worker-2ee8usp3 |
Comm: tcp://10.131.9.55:42358 | Total threads: 9 |
Dashboard: http://10.131.9.55:41858/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:42166 | |
Local directory: /tmp/dask-worker-space/worker-8xaet4h_ |
Comm: tcp://10.131.9.55:41898 | Total threads: 9 |
Dashboard: http://10.131.9.55:46577/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:43498 | |
Local directory: /tmp/dask-worker-space/worker-1f1tuhu8 |
Comm: tcp://10.131.9.55:43676 | Total threads: 9 |
Dashboard: http://10.131.9.55:43961/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:33450 | |
Local directory: /tmp/dask-worker-space/worker-ks5_31yl |
Comm: tcp://10.131.9.55:38143 | Total threads: 9 |
Dashboard: http://10.131.9.55:35202/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:43135 | |
Local directory: /tmp/dask-worker-space/worker-yixqfswj |
Comm: tcp://10.131.9.55:37846 | Total threads: 9 |
Dashboard: http://10.131.9.55:41688/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:36043 | |
Local directory: /tmp/dask-worker-space/worker-zlkrbhuc |
Comm: tcp://10.131.9.55:42075 | Total threads: 9 |
Dashboard: http://10.131.9.55:38366/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:46157 | |
Local directory: /tmp/dask-worker-space/worker-h0pbjh4q |
Comm: tcp://10.131.9.55:44697 | Total threads: 9 |
Dashboard: http://10.131.9.55:33883/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:37429 | |
Local directory: /tmp/dask-worker-space/worker-ni9av1rc |
Comm: tcp://10.131.9.55:36879 | Total threads: 9 |
Dashboard: http://10.131.9.55:35875/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:42139 | |
Local directory: /tmp/dask-worker-space/worker-luuohaso |
Comm: tcp://10.131.9.55:35398 | Total threads: 9 |
Dashboard: http://10.131.9.55:34500/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:33062 | |
Local directory: /tmp/dask-worker-space/worker-1ydm1n0_ |
Comm: tcp://10.131.9.55:43955 | Total threads: 9 |
Dashboard: http://10.131.9.55:37569/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:40883 | |
Local directory: /tmp/dask-worker-space/worker-vj8me5c4 |
Comm: tcp://10.131.9.55:38198 | Total threads: 9 |
Dashboard: http://10.131.9.55:38505/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:36971 | |
Local directory: /tmp/dask-worker-space/worker-64ishubf |
Comm: tcp://10.131.9.55:43202 | Total threads: 9 |
Dashboard: http://10.131.9.55:38561/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:40325 | |
Local directory: /tmp/dask-worker-space/worker-h5qg7uww |
Comm: tcp://10.131.9.55:42179 | Total threads: 9 |
Dashboard: http://10.131.9.55:41412/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:35546 | |
Local directory: /tmp/dask-worker-space/worker-eilly2_9 |
Comm: tcp://10.131.9.55:33708 | Total threads: 9 |
Dashboard: http://10.131.9.55:40978/status | Memory: 15.30 GiB |
Nanny: tcp://10.131.9.55:35917 | |
Local directory: /tmp/dask-worker-space/worker-hu2dgdxo |
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
MLD_2D | gridT-2D.samldr1_1 | data.samldr1_1 | ALL | maps | Blues | (0,80) | m | M-5 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True lazy= True ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: nested concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}} 0 read gridT-2D ['samldr1_1'] lazy= True using load_data_xios reading gridT-2D using load_data_xios reading {'name': 'data_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO outputs from different xios server', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'name of config', 'type': 'str', 'default': '/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d'}, {'name': 'fileexp', 'description': 'name of config', 'type': 'str', 'default': 'SEDNA-DELTA'}, {'name': 'month', 'description': 'running number 2 digit', 'type': 'str', 'default': '02'}, {'name': 'freq', 'description': '1d or 1m', 'type': 'str', 'default': '1d'}, {'name': 'year', 'description': 'last digits of yearmonthdate.', 'type': 'str', 'default': '2012'}, {'name': 'file', 'description': 'file name', 'type': 'str', 'default': 'icemod'}, {'name': 'eio', 'description': 'xios mpi number', 'type': 'str', 'default': '0[0-5][0-9][0-9]'}], 'metadata': {}, 'args': {'urlpath': '{{path}}/{{year}}/{{month}}/*{{file}}_*_{{eio}}.nc', 'combine': 'nested', 'concat_dim': 'time_counter,x,y'}} took 196.62669324874878 seconds 0 merging gridT-2D ['samldr1_1'] param nav_lat will be included in data param nav_lon will be included in data param mask2d will be included in data ychunk= 10 calldatas_y_rechunk sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12) start rechunking with (130, 122, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 48) end of y_rechunk CPU times: user 58.7 s, sys: 9.17 s, total: 1min 7s Wall time: 3min 38s
<xarray.Dataset> Dimensions: (t: 61, y: 6540, x: 6560) Coordinates: * t (t) object 2012-03-01 12:00:00 ... 2012-04-30 12:00:00 * y (y) int64 1 2 3 4 5 6 7 ... 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 ... 6555 6556 6557 6558 6559 6560 nav_lat (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray> time_centered (t) object dask.array<chunksize=(61,), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray> Data variables: samldr1_1 (t, y, x) float32 dask.array<chunksize=(61, 130, 6560), meta=np.ndarray> Attributes: (12/26) name: /ccc/scratch/cont003/ra5563/talandel/ONGOING-RUN... description: ocean T grid variables title: ocean T grid variables Conventions: CF-1.6 timeStamp: 2022-Jan-19 12:40:29 GMT uuid: 0c903256-6713-4218-8157-b00d89bc0866 ... ... start_date: 20090101 output_frequency: 1d CONFIG: SEDNA CASE: DELTA history: Thu Jan 20 11:49:42 2022: ncks -4 -L 1 SEDNA-DEL... NCO: netCDF Operators version 4.9.1 (Homepage = http:...
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
#calc= True #save= False #plot= True Value='MLD_2D' Zone='ALL' Plot='maps' cmap='Blues' clabel='m' clim= (0, 80) outputpath='../results/SEDNA_DELTA_MONITOR/' nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/' filename='SEDNA_maps_ALL_MLD_2D' data=monitor.optimize_dataset(data) #3 Start computing data= data.samldr1_1 monitor.optimize_dataset(data) add optimise here once otimise can recognise
<xarray.DataArray 'samldr1_1' (t: 61, y: 6540, x: 6560)> dask.array<rechunk-merge, shape=(61, 6540, 6560), dtype=float32, chunksize=(61, 130, 6560), chunktype=numpy.ndarray> Coordinates: * t (t) object 2012-03-01 12:00:00 ... 2012-04-30 12:00:00 * y (y) int64 1 2 3 4 5 6 7 ... 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 ... 6555 6556 6557 6558 6559 6560 nav_lat (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray> time_centered (t) object dask.array<chunksize=(61,), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray> Attributes: standard_name: ocean_mixed_layer_thickness_defined_by_sigma_theta long_name: Mixed Layer Depth (dsigma = 0.01 wrt 1m) units: m online_operation: average interval_operation: 40 s interval_write: 1 d cell_methods: time: mean (interval: 40 s)
#5 Plotting filename= plots.maps(data,path=outputpath,filename=filename,save=savefig,cmap=cmap,clim=clim,clabel=clabel) ../results/SEDNA_DELTA_MONITOR/SEDNA_maps_ALL_MLD_2D_20120301-20120430.html starts plotting
WARNING:param.dynamic_operation: Callable raised "KilledWorker("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2012, 3, 1, 12, 0, 0, 0, has_year_zero=True)) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2012, 3, 1, 12, 0, 0, 0, has_year_zero=True), height=400, scale=1.0, width=400, x_range=None, y_range=None) WARNING:param.dynamic_operation: Callable raised "KilledWorker("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)". Invoked as dynamic_operation(cftime.DatetimeNoLeap(2012, 3, 1, 12, 0, 0, 0, has_year_zero=True), height=400, scale=1.0, width=400, x_range=None, y_range=None) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2012, 3, 1, 12, 0, 0, 0, has_year_zero=True)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2012, 3, 1, 12, 0, 0, 0, has_year_zero=True)) WARNING:param.dynamic_mul: Callable raised "KilledWorker("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)". Invoked as dynamic_mul(cftime.DatetimeNoLeap(2012, 3, 1, 12, 0, 0, 0, has_year_zero=True))
--------------------------------------------------------------------------- KilledWorker Traceback (most recent call last) File <timed eval>:1, in <module> File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py:98, in auto(df, val, savefig, daskreport, outputpath, file_exp) 96 print('filename=',command ) 97 with performance_report(filename=daskreport+"_plot_"+step.Value+".html"): ---> 98 filename=eval(command ) 99 print(filename,'created') 100 display(IFrame(filename, width=1000, height=500)) File <string>:1, in <module> File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/plots.py:256, in maps(data, path, filename, save, cmap, clim, clabel) 254 plot = gf.land * plot * gf.coastline * gf.grid # default is 110m. 255 if save : --> 256 filename=save_html(plot,(path+filename),data.t) 257 #else: 258 #display(plot) 259 return filename File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/plots.py:57, in save_html(html, filename, t) 55 filename=filename + tname +'.html' 56 print(filename,'starts plotting') ---> 57 hvplot.save(html, filename, resources=INLINE) 58 return filename File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:772, in save(obj, filename, fmt, backend, resources, toolbar, title, **kwargs) 770 if formats[-1] in supported: 771 filename = '.'.join(formats[:-1]) --> 772 return renderer_obj.save(obj, filename, fmt=fmt, resources=resources, 773 title=title) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/plotting/renderer.py:592, in Renderer.save(self_or_cls, obj, basename, fmt, key, info, options, resources, title, **kwargs) 590 if fmt in MIME_TYPES: 591 basename = '.'.join([basename, fmt]) --> 592 plot.layout.save(basename, embed=True, resources=resources, title=title) 593 return 595 rendered = self_or_cls(plot, fmt) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/viewable.py:821, in Viewable.save(self, filename, title, resources, template, template_variables, embed, max_states, max_opts, embed_json, json_prefix, save_path, load_path, progress, embed_states, as_png, **kwargs) 778 def save(self, filename: str | IO, title: Optional[str] = None, 779 resources=None, template=None, 780 template_variables=None, embed=False, max_states=1000, 781 max_opts=3, embed_json=False, json_prefix='', save_path='./', 782 load_path=None, progress=True, embed_states={}, as_png=None, 783 **kwargs): 784 """ 785 Saves Panel objects to file. 786 (...) 819 string and ends with png. 820 """ --> 821 return save(self, filename, title, resources, template, 822 template_variables, embed, max_states, max_opts, 823 embed_json, json_prefix, save_path, load_path, 824 progress, embed_states, as_png, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/io/save.py:254, in save(panel, filename, title, resources, template, template_variables, embed, max_states, max_opts, embed_json, json_prefix, save_path, load_path, progress, embed_states, as_png, **kwargs) 252 model = doc 253 else: --> 254 model = panel.get_root(doc, comm) 255 if embed: 256 embed_state( 257 panel, model, doc, max_states, max_opts, embed_json, 258 json_prefix, save_path, load_path, progress, embed_states 259 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/viewable.py:558, in Renderable.get_root(self, doc, comm, preprocess) 541 """ 542 Returns the root model and applies pre-processing hooks 543 (...) 555 Returns the bokeh model corresponding to this panel object 556 """ 557 doc = init_doc(doc) --> 558 root = self._get_model(doc, comm=comm) 559 if preprocess: 560 self._preprocess(root) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/layout/base.py:146, in Panel._get_model(self, doc, root, parent, comm) 144 if root is None: 145 root = model --> 146 objects = self._get_objects(model, [], doc, root, comm) 147 props = dict(self._init_params(), objects=objects) 148 model.update(**self._process_param_change(props)) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/layout/base.py:131, in Panel._get_objects(self, model, old_objects, doc, root, comm) 129 else: 130 try: --> 131 child = pane._get_model(doc, root, model, comm) 132 except RerenderError: 133 return self._get_objects(model, current_objects[:i], doc, root, comm) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/pane/holoviews.py:248, in HoloViews._get_model(self, doc, root, parent, comm) 246 plot = self.object 247 else: --> 248 plot = self._render(doc, comm, root) 250 plot.pane = self 251 backend = plot.renderer.backend File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/panel/pane/holoviews.py:324, in HoloViews._render(self, doc, comm, root) 321 if comm: 322 kwargs['comm'] = comm --> 324 return renderer.get_plot(self.object, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/plotting/bokeh/renderer.py:70, in BokehRenderer.get_plot(self_or_cls, obj, doc, renderer, **kwargs) 63 @bothmethod 64 def get_plot(self_or_cls, obj, doc=None, renderer=None, **kwargs): 65 """ 66 Given a HoloViews Viewable return a corresponding plot instance. 67 Allows supplying a document attach the plot to, useful when 68 combining the bokeh model with another plot. 69 """ ---> 70 plot = super().get_plot(obj, doc, renderer, **kwargs) 71 if plot.document is None: 72 plot.document = Document() if self_or_cls.notebook_context else curdoc() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/plotting/renderer.py:217, in Renderer.get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs) 214 raise SkipRendering(msg.format(dims=dims)) 216 # Initialize DynamicMaps with first data item --> 217 initialize_dynamic(obj) 219 if not renderer: 220 renderer = self_or_cls File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/plotting/util.py:256, in initialize_dynamic(obj) 254 continue 255 if not len(dmap): --> 256 dmap[dmap._initial_key()] File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:1217, in DynamicMap.__getitem__(self, key) 1215 # Not a cross product and nothing cached so compute element. 1216 if cache is not None: return cache -> 1217 val = self._execute_callback(*tuple_key) 1218 if data_slice: 1219 val = self._dataslice(val, data_slice) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:984, in DynamicMap._execute_callback(self, *args) 981 kwargs['_memoization_hash_'] = hash_items 983 with dynamicmap_memoization(self.callback, self.streams): --> 984 retval = self.callback(*args, **kwargs) 985 return self._style(retval) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:582, in Callable.__call__(self, *args, **kwargs) 579 args, kwargs = (), dict(pos_kwargs, **kwargs) 581 try: --> 582 ret = self.callable(*args, **kwargs) 583 except KeyError: 584 # KeyError is caught separately because it is used to signal 585 # invalid keys on DynamicMap and should not warn 586 raise File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:283, in HoloMap.__mul__.<locals>.dynamic_mul(*args, **kwargs) 282 def dynamic_mul(*args, **kwargs): --> 283 element = self[args] 284 if reverse: 285 return other * element File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:1217, in DynamicMap.__getitem__(self, key) 1215 # Not a cross product and nothing cached so compute element. 1216 if cache is not None: return cache -> 1217 val = self._execute_callback(*tuple_key) 1218 if data_slice: 1219 val = self._dataslice(val, data_slice) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:984, in DynamicMap._execute_callback(self, *args) 981 kwargs['_memoization_hash_'] = hash_items 983 with dynamicmap_memoization(self.callback, self.streams): --> 984 retval = self.callback(*args, **kwargs) 985 return self._style(retval) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:582, in Callable.__call__(self, *args, **kwargs) 579 args, kwargs = (), dict(pos_kwargs, **kwargs) 581 try: --> 582 ret = self.callable(*args, **kwargs) 583 except KeyError: 584 # KeyError is caught separately because it is used to signal 585 # invalid keys on DynamicMap and should not warn 586 raise File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:283, in HoloMap.__mul__.<locals>.dynamic_mul(*args, **kwargs) 282 def dynamic_mul(*args, **kwargs): --> 283 element = self[args] 284 if reverse: 285 return other * element File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:1217, in DynamicMap.__getitem__(self, key) 1215 # Not a cross product and nothing cached so compute element. 1216 if cache is not None: return cache -> 1217 val = self._execute_callback(*tuple_key) 1218 if data_slice: 1219 val = self._dataslice(val, data_slice) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:984, in DynamicMap._execute_callback(self, *args) 981 kwargs['_memoization_hash_'] = hash_items 983 with dynamicmap_memoization(self.callback, self.streams): --> 984 retval = self.callback(*args, **kwargs) 985 return self._style(retval) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:582, in Callable.__call__(self, *args, **kwargs) 579 args, kwargs = (), dict(pos_kwargs, **kwargs) 581 try: --> 582 ret = self.callable(*args, **kwargs) 583 except KeyError: 584 # KeyError is caught separately because it is used to signal 585 # invalid keys on DynamicMap and should not warn 586 raise File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/overlay.py:34, in Overlayable.__mul__.<locals>.dynamic_mul(*args, **kwargs) 33 def dynamic_mul(*args, **kwargs): ---> 34 element = other[args] 35 return self * element File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:1217, in DynamicMap.__getitem__(self, key) 1215 # Not a cross product and nothing cached so compute element. 1216 if cache is not None: return cache -> 1217 val = self._execute_callback(*tuple_key) 1218 if data_slice: 1219 val = self._dataslice(val, data_slice) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:984, in DynamicMap._execute_callback(self, *args) 981 kwargs['_memoization_hash_'] = hash_items 983 with dynamicmap_memoization(self.callback, self.streams): --> 984 retval = self.callback(*args, **kwargs) 985 return self._style(retval) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:582, in Callable.__call__(self, *args, **kwargs) 579 args, kwargs = (), dict(pos_kwargs, **kwargs) 581 try: --> 582 ret = self.callable(*args, **kwargs) 583 except KeyError: 584 # KeyError is caught separately because it is used to signal 585 # invalid keys on DynamicMap and should not warn 586 raise File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:996, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs) 995 def dynamic_operation(*key, **kwargs): --> 996 key, obj = resolve(key, kwargs) 997 return apply(obj, *key, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:985, in Dynamic._dynamic_operation.<locals>.resolve(key, kwargs) 983 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key: 984 key = tuple(kwargs[k] for k in map_obj._posarg_keys) --> 985 return key, map_obj[key] File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:1217, in DynamicMap.__getitem__(self, key) 1215 # Not a cross product and nothing cached so compute element. 1216 if cache is not None: return cache -> 1217 val = self._execute_callback(*tuple_key) 1218 if data_slice: 1219 val = self._dataslice(val, data_slice) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:984, in DynamicMap._execute_callback(self, *args) 981 kwargs['_memoization_hash_'] = hash_items 983 with dynamicmap_memoization(self.callback, self.streams): --> 984 retval = self.callback(*args, **kwargs) 985 return self._style(retval) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:582, in Callable.__call__(self, *args, **kwargs) 579 args, kwargs = (), dict(pos_kwargs, **kwargs) 581 try: --> 582 ret = self.callable(*args, **kwargs) 583 except KeyError: 584 # KeyError is caught separately because it is used to signal 585 # invalid keys on DynamicMap and should not warn 586 raise File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:996, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs) 995 def dynamic_operation(*key, **kwargs): --> 996 key, obj = resolve(key, kwargs) 997 return apply(obj, *key, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:985, in Dynamic._dynamic_operation.<locals>.resolve(key, kwargs) 983 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key: 984 key = tuple(kwargs[k] for k in map_obj._posarg_keys) --> 985 return key, map_obj[key] File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:1217, in DynamicMap.__getitem__(self, key) 1215 # Not a cross product and nothing cached so compute element. 1216 if cache is not None: return cache -> 1217 val = self._execute_callback(*tuple_key) 1218 if data_slice: 1219 val = self._dataslice(val, data_slice) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:984, in DynamicMap._execute_callback(self, *args) 981 kwargs['_memoization_hash_'] = hash_items 983 with dynamicmap_memoization(self.callback, self.streams): --> 984 retval = self.callback(*args, **kwargs) 985 return self._style(retval) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/spaces.py:582, in Callable.__call__(self, *args, **kwargs) 579 args, kwargs = (), dict(pos_kwargs, **kwargs) 581 try: --> 582 ret = self.callable(*args, **kwargs) 583 except KeyError: 584 # KeyError is caught separately because it is used to signal 585 # invalid keys on DynamicMap and should not warn 586 raise File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:997, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs) 995 def dynamic_operation(*key, **kwargs): 996 key, obj = resolve(key, kwargs) --> 997 return apply(obj, *key, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:989, in Dynamic._dynamic_operation.<locals>.apply(element, *key, **kwargs) 987 def apply(element, *key, **kwargs): 988 kwargs = dict(util.resolve_dependent_kwargs(self.p.kwargs), **kwargs) --> 989 processed = self._process(element, key, kwargs) 990 if (self.p.link_dataset and isinstance(element, Dataset) and 991 isinstance(processed, Dataset) and processed._dataset is None): 992 processed._dataset = element.dataset File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/util/__init__.py:971, in Dynamic._process(self, element, key, kwargs) 969 elif isinstance(self.p.operation, Operation): 970 kwargs = {k: v for k, v in kwargs.items() if k in self.p.operation.param} --> 971 return self.p.operation.process_element(element, key, **kwargs) 972 else: 973 return self.p.operation(element, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/operation.py:194, in Operation.process_element(self, element, key, **params) 191 else: 192 self.p = param.ParamOverrides(self, params, 193 allow_extra_keywords=self._allow_extra_keywords) --> 194 return self._apply(element, key) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/operation.py:141, in Operation._apply(self, element, key) 139 if not in_method: 140 element._in_method = True --> 141 ret = self._process(element, key) 142 if hasattr(element, '_in_method') and not in_method: 143 element._in_method = in_method File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/geoviews/operation/projection.py:439, in project._process(self, element, key) 437 def _process(self, element, key=None): 438 for op in self._operations: --> 439 element = element.map(op.instance(projection=self.p.projection), 440 op.supported_types) 441 return element File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/__init__.py:204, in PipelineMeta.pipelined.<locals>.pipelined_fn(*args, **kwargs) 201 inst._in_method = True 203 try: --> 204 result = method_fn(*args, **kwargs) 205 if PipelineMeta.disable: 206 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/__init__.py:1216, in Dataset.map(self, *args, **kwargs) 1214 @wraps(LabelledData.map) 1215 def map(self, *args, **kwargs): -> 1216 return super(Dataset, self).map(*args, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/dimension.py:700, in LabelledData.map(self, map_fn, specs, clone) 698 return deep_mapped 699 else: --> 700 return map_fn(self) if applies else self File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/operation.py:220, in Operation.__call__(self, element, **kwargs) 218 kwargs['link_dataset'] = self._propagate_dataset 219 kwargs['link_inputs'] = self.p.link_inputs --> 220 return element.apply(self, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/accessors.py:45, in AccessorPipelineMeta.pipelined.<locals>.pipelined_call(*args, **kwargs) 42 inst._obj._in_method = True 44 try: ---> 45 result = __call__(*args, **kwargs) 47 if not in_method: 48 init_op = factory.instance( 49 output_type=type(inst), 50 kwargs={'mode': getattr(inst, 'mode', None)}, 51 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/accessors.py:202, in Apply.__call__(self, apply_function, streams, link_inputs, link_dataset, dynamic, per_element, **kwargs) 200 if hasattr(apply_function, 'dynamic'): 201 inner_kwargs['dynamic'] = False --> 202 new_obj = apply_function(self._obj, **inner_kwargs) 203 if (link_dataset and isinstance(self._obj, Dataset) and 204 isinstance(new_obj, Dataset) and new_obj._dataset is None): 205 new_obj._dataset = self._obj.dataset File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/operation.py:214, in Operation.__call__(self, element, **kwargs) 210 return element.clone([(k, self._apply(el, key=k)) 211 for k, el in element.items()]) 212 elif ((self._per_element and isinstance(element, Element)) or 213 (not self._per_element and isinstance(element, ViewableElement))): --> 214 return self._apply(element) 215 elif 'streams' not in kwargs: 216 kwargs['streams'] = self.p.streams File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/operation.py:141, in Operation._apply(self, element, key) 139 if not in_method: 140 element._in_method = True --> 141 ret = self._process(element, key) 142 if hasattr(element, '_in_method') and not in_method: 143 element._in_method = in_method File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/geoviews/operation/projection.py:40, in _project_operation._process(self, element, key) 39 def _process(self, element, key=None): ---> 40 return element.map(self._process_element, self.supported_types) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/__init__.py:204, in PipelineMeta.pipelined.<locals>.pipelined_fn(*args, **kwargs) 201 inst._in_method = True 203 try: --> 204 result = method_fn(*args, **kwargs) 205 if PipelineMeta.disable: 206 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/__init__.py:1216, in Dataset.map(self, *args, **kwargs) 1214 @wraps(LabelledData.map) 1215 def map(self, *args, **kwargs): -> 1216 return super(Dataset, self).map(*args, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/dimension.py:700, in LabelledData.map(self, map_fn, specs, clone) 698 return deep_mapped 699 else: --> 700 return map_fn(self) if applies else self File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/geoviews/operation/projection.py:239, in project_quadmesh._process_element(self, element) 235 proj = self.p.projection 236 irregular = any(element.interface.irregular(element, kd) 237 for kd in element.kdims) --> 239 zs = element.dimension_values(2, flat=False) 240 if irregular: 241 X, Y = [np.asarray(element.interface.coords( 242 element, kd, expanded=True, edges=False)) 243 for kd in element.kdims] File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/__init__.py:204, in PipelineMeta.pipelined.<locals>.pipelined_fn(*args, **kwargs) 201 inst._in_method = True 203 try: --> 204 result = method_fn(*args, **kwargs) 205 if PipelineMeta.disable: 206 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/__init__.py:1102, in Dataset.dimension_values(self, dimension, expanded, flat) 1086 """Return the values along the requested dimension. 1087 1088 Args: (...) 1099 NumPy array of values along the requested dimension 1100 """ 1101 dim = self.get_dimension(dimension, strict=True) -> 1102 values = self.interface.values(self, dim, expanded, flat) 1103 if dim.nodata is not None: 1104 # Ensure nodata applies to boolean data in py2 1105 values = np.where(values==dim.nodata, np.NaN, values) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/holoviews/core/data/xarray.py:415, in XArrayInterface.values(cls, dataset, dim, expanded, flat, compute, keep_index) 413 da = dask_array_module() 414 if compute and da and isinstance(data, da.Array): --> 415 data = data.compute() 416 if is_cupy(data): 417 import cupy File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/base.py:315, in DaskMethodsMixin.compute(self, **kwargs) 291 def compute(self, **kwargs): 292 """Compute this dask collection 293 294 This turns a lazy Dask collection into its in-memory equivalent. (...) 313 dask.base.compute 314 """ --> 315 (result,) = compute(self, traverse=False, **kwargs) 316 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/base.py:598, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs) 595 keys.append(x.__dask_keys__()) 596 postcomputes.append(x.__dask_postcompute__()) --> 598 results = schedule(dsk, keys, **kwargs) 599 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:3001, in Client.get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2999 should_rejoin = False 3000 try: -> 3001 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 3002 finally: 3003 for f in futures.values(): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2175, in Client.gather(self, futures, errors, direct, asynchronous) 2173 else: 2174 local_worker = None -> 2175 return self.sync( 2176 self._gather, 2177 futures, 2178 errors=errors, 2179 direct=direct, 2180 local_worker=local_worker, 2181 asynchronous=asynchronous, 2182 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:338, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 336 return future 337 else: --> 338 return sync( 339 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 340 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:405, in sync(loop, func, callback_timeout, *args, **kwargs) 403 if error: 404 typ, exc, tb = error --> 405 raise exc.with_traceback(tb) 406 else: 407 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:378, in sync.<locals>.f() 376 future = asyncio.wait_for(future, callback_timeout) 377 future = asyncio.ensure_future(future) --> 378 result = yield future 379 except Exception: 380 error = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/tornado/gen.py:762, in Runner.run(self) 759 exc_info = None 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2038, in Client._gather(self, futures, errors, direct, local_worker) 2036 exc = CancelledError(key) 2037 else: -> 2038 raise exception.with_traceback(traceback) 2039 raise exc 2040 if errors == "skip": KilledWorker: ("('where-212e8886a55c8d194dba9332f598a8e4', 0, 0, 0)", <WorkerState 'tcp://10.131.9.55:41898', name: SLURMCluster-1-1, status: closed, memory: 0, processing: 1>)