%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/monitor.py'>
If you submit the job with job scheduler; below are list of enviroment variable one can pass
local : if True run dask local cluster, if not true, put number of workers setted in the 'local' if no 'local ' given, local will be setted automatically to 'True'
%env ychunk='2', #%env tchunk='2'
controls chunk. 'False' sets no modification from original netcdf file's chunk.
ychunk=10 will group the original netcdf file to 10 by 10
tchunk=1 will chunk the time coordinate one by one
%env file_exp=
'file_exp': Which 'experiment' name is it? this corresopnds to intake catalog name without path and .yaml
#%env year=
for Validation, this correspoinds to path/year/month 's year for monitoring, this corresponids to 'date' having means do all files in the monitoring directory setting it as 0[0-9] &1[0-9]& [2-3][0-9], the job can be separated in three lots. For DELTA experiment, year corresponds to really 'year'
%env month=
for monitoring this corresponds to file path path-XIOS.{month}/
For DELTA experiment, year corresponds to really 'month'
proceed saving? True or False , Default is setted as True
proceed plotting? True or False , Default is setted as True
proceed computation? or just load computed result? True or False , Default is setted as True
save output file used for plotting
using kerchunked file -> False, not using kerhcunk -> True
name of control file to be used for computation/plots/save/ We have number of M_xxx.csv
Monitor.sh calls M_MLD_2D
and AWTD.sh, Fluxnet.sh, Siconc.sh, IceClim.sh, FWC_SSH.sh, Integrals.sh , Sections.sh
M_AWTMD
M_Fluxnet
M_Ice_quantities
M_IceClim M_IceConce M_IceThick
M_FWC_2D M_FWC_integrals M_FWC_SSH M_SSH_anomaly
M_Mean_temp_velo M_Mooring
M_Sectionx M_Sectiony
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene8000.c-irene.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False tgcc local cluster starting This code is running on irene8000.c-irene.tgcc.ccc.cea.fr using SEDNA_DELTA_MONITOR file experiment, read from ../lib/SEDNA_DELTA_MONITOR.yaml on year= 2013 on month= 06 outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6570265irene8000.c-irene.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_06M_IceConce/ CPU times: user 526 ms, sys: 157 ms, total: 682 ms Wall time: 14.9 s
Client-456a5acd-2483-11ed-b972-080038bfd9c6
Connection method: Cluster object | Cluster type: distributed.LocalCluster |
Dashboard: http://127.0.0.1:8787/status |
64cdf726
Dashboard: http://127.0.0.1:8787/status | Workers: 12 |
Total threads: 48 | Total memory: 2.86 TiB |
Status: running | Using processes: True |
Scheduler-e48b9b15-29a3-404c-b421-612c803a31a3
Comm: tcp://127.0.0.1:37818 | Workers: 12 |
Dashboard: http://127.0.0.1:8787/status | Total threads: 48 |
Started: Just now | Total memory: 2.86 TiB |
Comm: tcp://127.0.0.1:33477 | Total threads: 4 |
Dashboard: http://127.0.0.1:46295/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:44487 | |
Local directory: /tmp/dask-worker-space/worker-_subp3aw |
Comm: tcp://127.0.0.1:38711 | Total threads: 4 |
Dashboard: http://127.0.0.1:42978/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:41208 | |
Local directory: /tmp/dask-worker-space/worker-p6ollexb |
Comm: tcp://127.0.0.1:35593 | Total threads: 4 |
Dashboard: http://127.0.0.1:38224/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:44810 | |
Local directory: /tmp/dask-worker-space/worker-qnhh6aaf |
Comm: tcp://127.0.0.1:43368 | Total threads: 4 |
Dashboard: http://127.0.0.1:45102/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:45151 | |
Local directory: /tmp/dask-worker-space/worker-a4d1ckj4 |
Comm: tcp://127.0.0.1:34037 | Total threads: 4 |
Dashboard: http://127.0.0.1:41911/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:36497 | |
Local directory: /tmp/dask-worker-space/worker-fq1s4z_y |
Comm: tcp://127.0.0.1:35129 | Total threads: 4 |
Dashboard: http://127.0.0.1:36709/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:42868 | |
Local directory: /tmp/dask-worker-space/worker-tg9xr5fl |
Comm: tcp://127.0.0.1:34320 | Total threads: 4 |
Dashboard: http://127.0.0.1:45676/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:43171 | |
Local directory: /tmp/dask-worker-space/worker-ye9a0g1x |
Comm: tcp://127.0.0.1:36511 | Total threads: 4 |
Dashboard: http://127.0.0.1:35620/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:38299 | |
Local directory: /tmp/dask-worker-space/worker-em_w2z2e |
Comm: tcp://127.0.0.1:41905 | Total threads: 4 |
Dashboard: http://127.0.0.1:39860/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:40170 | |
Local directory: /tmp/dask-worker-space/worker-su5hcegy |
Comm: tcp://127.0.0.1:35157 | Total threads: 4 |
Dashboard: http://127.0.0.1:37410/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:37551 | |
Local directory: /tmp/dask-worker-space/worker-x1ns0ipx |
Comm: tcp://127.0.0.1:39481 | Total threads: 4 |
Dashboard: http://127.0.0.1:38815/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:43618 | |
Local directory: /tmp/dask-worker-space/worker-1dm_fv24 |
Comm: tcp://127.0.0.1:46652 | Total threads: 4 |
Dashboard: http://127.0.0.1:44517/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:41733 | |
Local directory: /tmp/dask-worker-space/worker-ldxmh25q |
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
IceConce | icemod.siconc | (data.siconc.where(data.siconc >0)).to_dataset... | ALL | maps | Blues | None | M-4 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True lazy= True ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: nested concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}} 0 read icemod ['siconc'] lazy= True using load_data_xios reading icemod using load_data_xios reading {'name': 'data_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO outputs from different xios server', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'name of config', 'type': 'str', 'default': '/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d'}, {'name': 'fileexp', 'description': 'name of config', 'type': 'str', 'default': 'SEDNA-DELTA'}, {'name': 'month', 'description': 'running number 2 digit', 'type': 'str', 'default': '02'}, {'name': 'freq', 'description': '1d or 1m', 'type': 'str', 'default': '1d'}, {'name': 'year', 'description': 'last digits of yearmonthdate.', 'type': 'str', 'default': '2012'}, {'name': 'file', 'description': 'file name', 'type': 'str', 'default': 'icemod'}, {'name': 'eio', 'description': 'xios mpi number', 'type': 'str', 'default': '0[0-5][0-9][0-9]'}], 'metadata': {}, 'args': {'urlpath': '{{path}}/{{year}}/{{month}}/*{{file}}_*_{{eio}}.nc', 'combine': 'nested', 'concat_dim': 'y'}}
HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3588096, sblock->base_addr = 0, stored_eof = 115111432 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 70184960, sblock->base_addr = 0, stored_eof = 121997013 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 4141056, sblock->base_addr = 0, stored_eof = 138561926 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3481600, sblock->base_addr = 0, stored_eof = 123998902 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 2: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 1033216, sblock->base_addr = 0, stored_eof = 52611074 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 78872576, sblock->base_addr = 0, stored_eof = 144577582 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 2: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3641344, sblock->base_addr = 0, stored_eof = 89949840 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 2: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3493888, sblock->base_addr = 0, stored_eof = 138106336 major: File accessibility minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 13791232, sblock->base_addr = 0, stored_eof = 90782283 major: File accessibility minor: File has been truncated 2022-08-25 16:36:36,271 - distributed.worker - WARNING - Compute Failed Key: open_dataset-8f1714c9-3e5b-4a92-bdc6-ef36c2b1a663 Function: execute_task args: ((<function apply at 0x2b1fafa51000>, <function open_dataset at 0x2b1fd9983250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0095.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 3: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 1182208, sblock->base_addr = 0, stored_eof = 115801892 major: File accessibility minor: File has been truncated 2022-08-25 16:36:36,291 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2edfcd27-1f46-42df-865f-759124a51f5e Function: execute_task args: ((<function apply at 0x2b2ddb9a1000>, <function open_dataset at 0x2b2e05885750>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0362.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 2: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3424256, sblock->base_addr = 0, stored_eof = 75704137 major: File accessibility minor: File has been truncated 2022-08-25 16:36:36,369 - distributed.worker - WARNING - Compute Failed Key: open_dataset-63baa918-b029-4cb4-b25c-572d352c8ba0 Function: execute_task args: ((<function apply at 0x2b42292f5000>, <function open_dataset at 0x2b425320a320>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0480.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,399 - distributed.worker - WARNING - Compute Failed Key: open_dataset-64995846-88fc-4142-8cf4-47ab2fd8629e Function: execute_task args: ((<function apply at 0x2b42292f5000>, <function open_dataset at 0x2b425320a320>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0257.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,419 - distributed.worker - WARNING - Compute Failed Key: open_dataset-a3695ce5-c330-4c27-924e-b70fa1bb4ac5 Function: execute_task args: ((<function apply at 0x2acf247dd000>, <function open_dataset at 0x2acf4e66feb0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0285.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,421 - distributed.worker - WARNING - Compute Failed Key: open_dataset-516e425a-020c-460a-85ff-5e6dbeea63ea Function: execute_task args: ((<function apply at 0x2b4b79191000>, <function open_dataset at 0x2b4ba30d7370>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0263.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,422 - distributed.worker - WARNING - Compute Failed Key: open_dataset-52f683aa-12ce-491d-ac7f-deef0a5f2806 Function: execute_task args: ((<function apply at 0x2b4b79191000>, <function open_dataset at 0x2b4ba30d7370>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0357.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,422 - distributed.worker - WARNING - Compute Failed Key: open_dataset-52a14169-9ad1-44d3-a607-410b686becad Function: execute_task args: ((<function apply at 0x2b4b79191000>, <function open_dataset at 0x2b4ba30d7370>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0337.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,423 - distributed.worker - WARNING - Compute Failed Key: open_dataset-51d00276-f107-4f8b-8c76-47313aa6e479 Function: execute_task args: ((<function apply at 0x2b4b79191000>, <function open_dataset at 0x2b4ba30d7370>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0284.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,579 - distributed.worker - WARNING - Compute Failed Key: open_dataset-e4d1c51c-ccf0-4ff8-8242-846c3c756742 Function: execute_task args: ((<function apply at 0x2ac96ec8d000>, <function open_dataset at 0x2ac998ba70a0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0179.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,593 - distributed.worker - WARNING - Compute Failed Key: open_dataset-cf64fa80-fb94-4055-af86-9f01b02685d0 Function: execute_task args: ((<function apply at 0x2b3449811000>, <function open_dataset at 0x2b347363b760>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0445.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,593 - distributed.worker - WARNING - Compute Failed Key: open_dataset-cc9b52ca-9183-4eb0-bb67-ed90039dc86d Function: execute_task args: ((<function apply at 0x2b3449811000>, <function open_dataset at 0x2b347363b760>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0475.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,593 - distributed.worker - WARNING - Compute Failed Key: open_dataset-cf57b7d2-c8f1-4f16-bd59-8ae2adcc0bdf Function: execute_task args: ((<function apply at 0x2b3449811000>, <function open_dataset at 0x2b347363b760>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0351.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,634 - distributed.worker - WARNING - Compute Failed Key: open_dataset-166ff3e3-ea20-4260-b724-6fac8a39f25c Function: execute_task args: ((<function apply at 0x2b62e7f1d000>, <function open_dataset at 0x2b6311e15d80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0375.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,635 - distributed.worker - WARNING - Compute Failed Key: open_dataset-167c63f4-bc06-41f8-a79e-507636cd1e78 Function: execute_task args: ((<function apply at 0x2b62e7f1d000>, <function open_dataset at 0x2b6311e15d80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0290.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,660 - distributed.worker - WARNING - Compute Failed Key: open_dataset-639ee4b0-798e-4ac3-b2f2-8e8dbf98b51b Function: execute_task args: ((<function apply at 0x2b42292f5000>, <function open_dataset at 0x2b425320a320>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0260.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,660 - distributed.worker - WARNING - Compute Failed Key: open_dataset-6375c356-d320-4715-b9e7-355181c7e9a3 Function: execute_task args: ((<function apply at 0x2b42292f5000>, <function open_dataset at 0x2b425320a320>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0388.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,705 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b6ced9d4-4472-411e-87dc-5299d4fd4b69 Function: execute_task args: ((<function apply at 0x2abbb63c9000>, <function open_dataset at 0x2abbe0271900>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0450.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:36,705 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b6c99e1d-b8e6-4c16-9d7e-988570e3b8fe Function: execute_task args: ((<function apply at 0x2abbb63c9000>, <function open_dataset at 0x2abbe0271900>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0316.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:36,705 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b71be7b1-26fd-4995-a1e9-4c7de27d0c9a Function: execute_task args: ((<function apply at 0x2abbb63c9000>, <function open_dataset at 0x2abbe0271900>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0334.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,086 - distributed.worker - WARNING - Compute Failed Key: open_dataset-3fc90b9a-05f3-4028-a3b8-52bdc3f85b49 Function: execute_task args: ((<function apply at 0x2b1fad185000>, <function open_dataset at 0x2b1fd709f370>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0472.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,113 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2c80ef4a-79f7-4392-9818-1a7cf8750478 Function: execute_task args: ((<function apply at 0x2b2ddb9a1000>, <function open_dataset at 0x2b2e05885750>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0097.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,113 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2e69c680-ac54-48ed-8f50-f17b05220ea5 Function: execute_task args: ((<function apply at 0x2b2ddb9a1000>, <function open_dataset at 0x2b2e05885750>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0182.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,113 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2d20e479-3bb9-4802-a2a0-00e7afa63a6b Function: execute_task args: ((<function apply at 0x2b2ddb9a1000>, <function open_dataset at 0x2b2e05885750>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0365.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,289 - distributed.worker - WARNING - Compute Failed Key: open_dataset-77ca7256-5fb4-4c14-a847-1a3e860590ea Function: execute_task args: ((<function apply at 0x2ad48bbb1000>, <function open_dataset at 0x2ad4b5af7130>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0099.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:49,409 - distributed.worker - WARNING - Compute Failed Key: open_dataset-8e530fbf-0307-4d6b-8e74-fbd6f71f30df Function: execute_task args: ((<function apply at 0x2b1fafa51000>, <function open_dataset at 0x2b1fd9983250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0261.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 16:36:49,409 - distributed.worker - WARNING - Compute Failed Key: open_dataset-8dfe8b5a-2321-4dc4-a6fb-65282ba8fac6 Function: execute_task args: ((<function apply at 0x2b1fafa51000>, <function open_dataset at 0x2b1fd9983250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0221.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3104768, sblock->base_addr = 0, stored_eof = 73629030 major: File accessibility minor: File has been truncated 2022-08-25 16:36:49,478 - distributed.worker - WARNING - Compute Failed Key: open_dataset-90b2bcb8-55f0-4f01-a49a-a71f5b56cf48 Function: execute_task args: ((<function apply at 0x2b1fafa51000>, <function open_dataset at 0x2b1fd9983250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0320.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,478 - distributed.worker - WARNING - Compute Failed Key: open_dataset-90d4395d-d305-41f6-a41d-dcb2672568d6 Function: execute_task args: ((<function apply at 0x2b1fafa51000>, <function open_dataset at 0x2b1fd9983250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0225.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 16:36:49,521 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2fbc33e6-62ef-4c49-b083-ea0e4199446d Function: execute_task args: ((<function apply at 0x2b2ddb9a1000>, <function open_dataset at 0x2b2e05885750>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0268.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 4542464, sblock->base_addr = 0, stored_eof = 133734280 major: File accessibility minor: File has been truncated 2022-08-25 16:36:49,603 - distributed.worker - WARNING - Compute Failed Key: open_dataset-7b26ce4e-cd05-4c82-9c40-3a30558bfb97 Function: execute_task args: ((<function apply at 0x2ad48bbb1000>, <function open_dataset at 0x2ad4b5af7130>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_icemod_201306-201306_0338.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['depthu_bounds', 'time_counter_bounds', 'sistre', 'snthic', 'uwspd10', 'vtau_iceoce', 'intstrx', 'intstry', 'iicestru', 'deptht_bounds', 'utau_iceoce', 'sozotaux', 'sometauy', 'sishea', 'iicestrv', 'botpres', 'snvolu', 'sisali', 'vwspd10', 'time_centered_bounds', 'sitemp', 'sidive', 'utau_atmoce', 'rhop_sig0', 'siages', 'mldkz5', 'vtau_atmoce'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')"
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) File <timed exec>:6, in <module> File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:681, in datas(catalog_url, dfi, month, year, daskreport, lazy) 676 datadict, paramdict = getdict(dfi) 677 #print('datadict:',datadict) 678 #if datadict == {}: 679 # data=0 680 #else: --> 681 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 682 for s in paramdict: 683 print('param',s,'will be included in data') File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:499, in outputs(catalog_url, datadict, month, year, daskreport, lazy) 496 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"): 497 #ds=load_data_xios_patch(cat,filename,month,catalog_url) 498 print("lazy=",lazy) --> 499 ds = load_data_xios(cat,filename,items,month,year) if ('True' in lazy) else load_data_xios_kerchunk(cat,filename,items,month,year,rome=True) 500 extime=time.time() - start 501 print(' took', extime, 'seconds') File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:374, in load_data_xios(cat, filename, items, month, year) 372 desc=cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).describe() 373 print('using load_data_xios reading ',desc) --> 374 ds = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask() 375 ds = ds[items] 376 return ds File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:69, in DataSourceMixin.to_dask(self) 67 def to_dask(self): 68 """Return xarray object where variables are dask arrays""" ---> 69 return self.read_chunked() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:44, in DataSourceMixin.read_chunked(self) 42 def read_chunked(self): 43 """Return xarray object (which will have chunks)""" ---> 44 self._load_metadata() 45 return self._ds File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake/source/base.py:236, in DataSourceBase._load_metadata(self) 234 """load metadata only if needed""" 235 if self._schema is None: --> 236 self._schema = self._get_schema() 237 self.dtype = self._schema.dtype 238 self.shape = self._schema.shape File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:18, in DataSourceMixin._get_schema(self) 15 self.urlpath = self._get_cache(self.urlpath)[0] 17 if self._ds is None: ---> 18 self._open_dataset() 20 metadata = { 21 'dims': dict(self._ds.dims), 22 'data_vars': {k: list(self._ds[k].coords) 23 for k in self._ds.data_vars.keys()}, 24 'coords': tuple(self._ds.coords.keys()), 25 } 26 if getattr(self, 'on_server', False): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/netcdf.py:92, in NetCDFSource._open_dataset(self) 88 else: 89 # https://github.com/intake/filesystem_spec/issues/476#issuecomment-732372918 90 url = fsspec.open(self.urlpath, **self.storage_options).open() ---> 92 self._ds = _open_dataset(url, chunks=self.chunks, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:980, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs) 975 datasets = [preprocess(ds) for ds in datasets] 977 if parallel: 978 # calling compute here will return the datasets/file_objs lists, 979 # the underlying datasets will still be stored as dask arrays --> 980 datasets, closers = dask.compute(datasets, closers) 982 # Combine all datasets, closing them in case of a ValueError 983 try: File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/base.py:598, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs) 595 keys.append(x.__dask_keys__()) 596 postcomputes.append(x.__dask_postcompute__()) --> 598 results = schedule(dsk, keys, **kwargs) 599 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:3001, in Client.get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2999 should_rejoin = False 3000 try: -> 3001 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 3002 finally: 3003 for f in futures.values(): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2175, in Client.gather(self, futures, errors, direct, asynchronous) 2173 else: 2174 local_worker = None -> 2175 return self.sync( 2176 self._gather, 2177 futures, 2178 errors=errors, 2179 direct=direct, 2180 local_worker=local_worker, 2181 asynchronous=asynchronous, 2182 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:338, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 336 return future 337 else: --> 338 return sync( 339 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 340 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:405, in sync(loop, func, callback_timeout, *args, **kwargs) 403 if error: 404 typ, exc, tb = error --> 405 raise exc.with_traceback(tb) 406 else: 407 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:378, in sync.<locals>.f() 376 future = asyncio.wait_for(future, callback_timeout) 377 future = asyncio.ensure_future(future) --> 378 result = yield future 379 except Exception: 380 error = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/tornado/gen.py:762, in Runner.run(self) 759 exc_info = None 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2038, in Client._gather(self, futures, errors, direct, local_worker) 2036 exc = CancelledError(key) 2037 else: -> 2038 raise exception.with_traceback(traceback) 2039 raise exc 2040 if errors == "skip": File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/utils.py:41, in apply() 39 def apply(func, args, kwargs=None): 40 if kwargs: ---> 41 return func(*args, **kwargs) 42 else: 43 return func(*args) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:515, in open_dataset() 512 kwargs.update(backend_kwargs) 514 if engine is None: --> 515 engine = plugins.guess_engine(filename_or_obj) 517 backend = plugins.get_backend(engine) 519 decoders = _resolve_decoders_kwargs( 520 decode_cf, 521 open_backend_dataset_parameters=backend.open_dataset_parameters, (...) 527 decode_coords=decode_coords, 528 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/plugins.py:155, in guess_engine() 147 else: 148 error_msg = ( 149 "found the following matches with the input file in xarray's IO " 150 f"backends: {compatible_engines}. But their dependencies may not be installed, see:\n" 151 "https://docs.xarray.dev/en/stable/user-guide/io.html \n" 152 "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html" 153 ) --> 155 raise ValueError(error_msg) ValueError: did not find a match in any of xarray's currently installed IO backends ['netcdf4', 'scipy', 'zarr']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see: https://docs.xarray.dev/en/stable/getting-started-guide/installing.html https://docs.xarray.dev/en/stable/user-guide/io.html
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) File <timed eval>:1, in <module> NameError: name 'data' is not defined