%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/monitor.py'>
If you submit the job with job scheduler; below are list of enviroment variable one can pass
local : if True run dask local cluster, if not true, put number of workers setted in the 'local' if no 'local ' given, local will be setted automatically to 'True'
%env ychunk='2', #%env tchunk='2'
controls chunk. 'False' sets no modification from original netcdf file's chunk.
ychunk=10 will group the original netcdf file to 10 by 10
tchunk=1 will chunk the time coordinate one by one
%env file_exp=
'file_exp': Which 'experiment' name is it? this corresopnds to intake catalog name without path and .yaml
#%env year=
for Validation, this correspoinds to path/year/month 's year for monitoring, this corresponids to 'date' having means do all files in the monitoring directory setting it as 0[0-9] &1[0-9]& [2-3][0-9], the job can be separated in three lots. For DELTA experiment, year corresponds to really 'year'
%env month=
for monitoring this corresponds to file path path-XIOS.{month}/
For DELTA experiment, year corresponds to really 'month'
proceed saving? True or False , Default is setted as True
proceed plotting? True or False , Default is setted as True
proceed computation? or just load computed result? True or False , Default is setted as True
save output file used for plotting
using kerchunked file -> False, not using kerhcunk -> True
name of control file to be used for computation/plots/save/ We have number of M_xxx.csv
Monitor.sh calls M_MLD_2D
and AWTD.sh, Fluxnet.sh, Siconc.sh, IceClim.sh, FWC_SSH.sh, Integrals.sh , Sections.sh
M_AWTMD
M_Fluxnet
M_Ice_quantities
M_IceClim M_IceConce M_IceThick
M_FWC_2D M_FWC_integrals M_FWC_SSH M_SSH_anomaly
M_Mean_temp_velo M_Mooring
M_Sectionx M_Sectiony
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene8000.c-irene.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False tgcc local cluster starting This code is running on irene8000.c-irene.tgcc.ccc.cea.fr using SEDNA_DELTA_MONITOR file experiment, read from ../lib/SEDNA_DELTA_MONITOR.yaml on year= 2013 on month= 07 outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6570280irene8000.c-irene.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_07M_SSH_anomaly/ CPU times: user 464 ms, sys: 153 ms, total: 617 ms Wall time: 14.4 s
Client-f8217bc9-2487-11ed-a351-080038bfd9c6
Connection method: Cluster object | Cluster type: distributed.LocalCluster |
Dashboard: http://127.0.0.1:8787/status |
5259427b
Dashboard: http://127.0.0.1:8787/status | Workers: 12 |
Total threads: 48 | Total memory: 2.86 TiB |
Status: running | Using processes: True |
Scheduler-c59c0443-f43e-460c-a6ca-c11634a604da
Comm: tcp://127.0.0.1:41803 | Workers: 12 |
Dashboard: http://127.0.0.1:8787/status | Total threads: 48 |
Started: Just now | Total memory: 2.86 TiB |
Comm: tcp://127.0.0.1:44818 | Total threads: 4 |
Dashboard: http://127.0.0.1:37443/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:46619 | |
Local directory: /tmp/dask-worker-space/worker-x1tw8icr |
Comm: tcp://127.0.0.1:36419 | Total threads: 4 |
Dashboard: http://127.0.0.1:39268/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:44489 | |
Local directory: /tmp/dask-worker-space/worker-01dja4ur |
Comm: tcp://127.0.0.1:39994 | Total threads: 4 |
Dashboard: http://127.0.0.1:41142/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:38336 | |
Local directory: /tmp/dask-worker-space/worker-qi27b5pk |
Comm: tcp://127.0.0.1:40396 | Total threads: 4 |
Dashboard: http://127.0.0.1:33520/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:41268 | |
Local directory: /tmp/dask-worker-space/worker-d4sl18mg |
Comm: tcp://127.0.0.1:40633 | Total threads: 4 |
Dashboard: http://127.0.0.1:46759/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:41771 | |
Local directory: /tmp/dask-worker-space/worker-jbp9mvw9 |
Comm: tcp://127.0.0.1:33584 | Total threads: 4 |
Dashboard: http://127.0.0.1:43770/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:39918 | |
Local directory: /tmp/dask-worker-space/worker-emomo8gc |
Comm: tcp://127.0.0.1:36143 | Total threads: 4 |
Dashboard: http://127.0.0.1:43653/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:45275 | |
Local directory: /tmp/dask-worker-space/worker-93c1euc9 |
Comm: tcp://127.0.0.1:42750 | Total threads: 4 |
Dashboard: http://127.0.0.1:33018/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:45768 | |
Local directory: /tmp/dask-worker-space/worker-28ovshqt |
Comm: tcp://127.0.0.1:38003 | Total threads: 4 |
Dashboard: http://127.0.0.1:42597/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:40898 | |
Local directory: /tmp/dask-worker-space/worker-_rk3qajm |
Comm: tcp://127.0.0.1:42692 | Total threads: 4 |
Dashboard: http://127.0.0.1:35080/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:46163 | |
Local directory: /tmp/dask-worker-space/worker-mqnbjbjf |
Comm: tcp://127.0.0.1:46170 | Total threads: 4 |
Dashboard: http://127.0.0.1:45581/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:40988 | |
Local directory: /tmp/dask-worker-space/worker-cew8cmdm |
Comm: tcp://127.0.0.1:46262 | Total threads: 4 |
Dashboard: http://127.0.0.1:34587/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:42865 | |
Local directory: /tmp/dask-worker-space/worker-9ebj_685 |
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
SSH_anomaly | gridT-2D.ssh | calc.SSH_anomaly(data) | ALL | maps | Spectral_r | (-0.35,0.35) | m | M-2 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True lazy= True ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: nested concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}} 0 read gridT-2D ['ssh'] lazy= True using load_data_xios reading gridT-2D using load_data_xios reading {'name': 'data_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO outputs from different xios server', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'name of config', 'type': 'str', 'default': '/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d'}, {'name': 'fileexp', 'description': 'name of config', 'type': 'str', 'default': 'SEDNA-DELTA'}, {'name': 'month', 'description': 'running number 2 digit', 'type': 'str', 'default': '02'}, {'name': 'freq', 'description': '1d or 1m', 'type': 'str', 'default': '1d'}, {'name': 'year', 'description': 'last digits of yearmonthdate.', 'type': 'str', 'default': '2012'}, {'name': 'file', 'description': 'file name', 'type': 'str', 'default': 'icemod'}, {'name': 'eio', 'description': 'xios mpi number', 'type': 'str', 'default': '0[0-5][0-9][0-9]'}], 'metadata': {}, 'args': {'urlpath': '{{path}}/{{year}}/{{month}}/*{{file}}_*_{{eio}}.nc', 'combine': 'nested', 'concat_dim': 'y'}}
HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 405504, sblock->base_addr = 0, stored_eof = 7375220 major: File accessibility minor: File has been truncated 2022-08-25 17:10:13,921 - distributed.worker - WARNING - Compute Failed Key: open_dataset-ca3fa81a-6cb2-412c-8e9d-7048aa618215 Function: execute_task args: ((<function apply at 0x2ab55fcc5000>, <function open_dataset at 0x2ab589b115a0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0269.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:13,923 - distributed.worker - WARNING - Compute Failed Key: open_dataset-9e4d8235-e39a-46ae-adbb-9527deafb7b9 Function: execute_task args: ((<function apply at 0x2b47047f9000>, <function open_dataset at 0x2b472e5c2b90>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0033.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:13,949 - distributed.worker - WARNING - Compute Failed Key: open_dataset-58daa32a-7567-4473-879a-5ca3de6a5313 Function: execute_task args: ((<function apply at 0x2ad63fded000>, <function open_dataset at 0x2ad669d20b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0009.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:13,957 - distributed.worker - WARNING - Compute Failed Key: open_dataset-58f916f7-6838-4235-a705-63e07609d33f Function: execute_task args: ((<function apply at 0x2ad63fded000>, <function open_dataset at 0x2ad669d20b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0342.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:13,979 - distributed.worker - WARNING - Compute Failed Key: open_dataset-730b9d50-31ea-4252-b052-f35cb528c942 Function: execute_task args: ((<function apply at 0x2b7fdd5bd000>, <function open_dataset at 0x2b80074b8790>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0068.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:13,984 - distributed.worker - WARNING - Compute Failed Key: open_dataset-8d0fbdb7-9aa9-4936-9054-bedd4c03562c Function: execute_task args: ((<function apply at 0x2b9f45c11000>, <function open_dataset at 0x2b9f6bb00940>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0038.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:13,986 - distributed.worker - WARNING - Compute Failed Key: open_dataset-734bdd18-8b62-4edf-8d39-c5218eb62844 Function: execute_task args: ((<function apply at 0x2b7fdd5bd000>, <function open_dataset at 0x2b80074b8790>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0055.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,222 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b4bdba0b-fb5f-4156-b833-fb7b9b34f50d Function: execute_task args: ((<function apply at 0x2b0f7d8dd000>, <function open_dataset at 0x2b0fa37d0550>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0236.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,222 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b4ed90f4-cab2-475f-88de-83b7e4aba0d8 Function: execute_task args: ((<function apply at 0x2b0f7d8dd000>, <function open_dataset at 0x2b0fa37d0550>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0007.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,223 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b50e9feb-ef1d-4e3e-9798-6175f5f5d430 Function: execute_task args: ((<function apply at 0x2b0f7d8dd000>, <function open_dataset at 0x2b0fa37d0550>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0174.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,223 - distributed.worker - WARNING - Compute Failed Key: open_dataset-ef6a8c29-22e2-44d0-a3bd-92986f6ff73d Function: execute_task args: ((<function apply at 0x2ab845879000>, <function open_dataset at 0x2ab86b7a8b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0201.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,223 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b46581db-0970-4112-af89-36a2f6baa0ec Function: execute_task args: ((<function apply at 0x2b0f7d8dd000>, <function open_dataset at 0x2b0fa37d0550>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0251.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,223 - distributed.worker - WARNING - Compute Failed Key: open_dataset-efbe9cf2-42e6-46c9-b21a-aebb221c87aa Function: execute_task args: ((<function apply at 0x2ab845879000>, <function open_dataset at 0x2ab86b7a8b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0192.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,224 - distributed.worker - WARNING - Compute Failed Key: open_dataset-f0d71e7b-5e89-47fd-aa57-3ed2c9a0e638 Function: execute_task args: ((<function apply at 0x2ab845879000>, <function open_dataset at 0x2ab86b7a8b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0238.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,225 - distributed.worker - WARNING - Compute Failed Key: open_dataset-effdec85-ee9b-4d16-9131-c45c7638fe63 Function: execute_task args: ((<function apply at 0x2ab845879000>, <function open_dataset at 0x2ab86b7a8b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0205.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,242 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2e8e0204-fe43-4f64-a0fb-25a4ac5f0db2 Function: execute_task args: ((<function apply at 0x2acd5d9b9000>, <function open_dataset at 0x2acd838a4700>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0014.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,242 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2ef94d6f-f72b-436b-b352-1621f30c612d Function: execute_task args: ((<function apply at 0x2acd5d9b9000>, <function open_dataset at 0x2acd838a4700>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0186.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,242 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2edc7563-a4b5-4989-8ded-1168306b4889 Function: execute_task args: ((<function apply at 0x2acd5d9b9000>, <function open_dataset at 0x2acd838a4700>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0158.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,251 - distributed.worker - WARNING - Compute Failed Key: open_dataset-6f2d56f8-a0ea-4516-b835-3649f854b164 Function: execute_task args: ((<function apply at 0x2b7fdd5bd000>, <function open_dataset at 0x2b80074b8790>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0149.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,253 - distributed.worker - WARNING - Compute Failed Key: open_dataset-70fe3d28-ef45-495e-9b2c-a6b97b89254d Function: execute_task args: ((<function apply at 0x2b7fdd5bd000>, <function open_dataset at 0x2b80074b8790>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0284.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,253 - distributed.worker - WARNING - Compute Failed Key: open_dataset-6f62e6fe-361b-45fd-a057-ae59f5430180 Function: execute_task args: ((<function apply at 0x2b7fdd5bd000>, <function open_dataset at 0x2b80074b8790>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0512.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:10:14,286 - distributed.worker - WARNING - Compute Failed Key: open_dataset-88c0a8c3-bde2-45ce-9d52-081fcb37a847 Function: execute_task args: ((<function apply at 0x2b9f45c11000>, <function open_dataset at 0x2b9f6bb00940>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0203.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,286 - distributed.worker - WARNING - Compute Failed Key: open_dataset-8c0a3da8-7a4e-4393-9036-abd8eba0647a Function: execute_task args: ((<function apply at 0x2b9f45c11000>, <function open_dataset at 0x2b9f6bb00940>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0075.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,287 - distributed.worker - WARNING - Compute Failed Key: open_dataset-8b9b3c9b-06b5-4901-b4c7-182b50f56865 Function: execute_task args: ((<function apply at 0x2b9f45c11000>, <function open_dataset at 0x2b9f6bb00940>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0335.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,290 - distributed.worker - WARNING - Compute Failed Key: open_dataset-474dbe6f-1198-40c0-a6be-9069a8f9bd3d Function: execute_task args: ((<function apply at 0x2aaeb4d05000>, <function open_dataset at 0x2aaedec08940>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0036.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,291 - distributed.worker - WARNING - Compute Failed Key: open_dataset-46ca63f5-0dbc-4930-a68f-1228354ee5e6 Function: execute_task args: ((<function apply at 0x2aaeb4d05000>, <function open_dataset at 0x2aaedec08940>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0290.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,432 - distributed.worker - WARNING - Compute Failed Key: open_dataset-dcc49a1d-d3d0-410c-b6d1-3dbfa02383e7 Function: execute_task args: ((<function apply at 0x2aba4c921000>, <function open_dataset at 0x2aba766d7490>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0372.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,433 - distributed.worker - WARNING - Compute Failed Key: open_dataset-dbf820b3-5781-419f-9837-b0e4021ade8a Function: execute_task args: ((<function apply at 0x2aba4c921000>, <function open_dataset at 0x2aba766d7490>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0194.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,433 - distributed.worker - WARNING - Compute Failed Key: open_dataset-dd565ea7-0c3c-434b-bd2e-388e01da4cf1 Function: execute_task args: ((<function apply at 0x2aba4c921000>, <function open_dataset at 0x2aba766d7490>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0098.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,504 - distributed.worker - WARNING - Compute Failed Key: open_dataset-58cf4602-e46b-4333-879a-8fe1a81f8c03 Function: execute_task args: ((<function apply at 0x2ad63fded000>, <function open_dataset at 0x2ad669d20b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0004.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:14,505 - distributed.worker - WARNING - Compute Failed Key: open_dataset-587b7c10-f249-4714-9e2c-fb454f719c4e Function: execute_task args: ((<function apply at 0x2ad63fded000>, <function open_dataset at 0x2ad669d20b80>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0090.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,596 - distributed.worker - WARNING - Compute Failed Key: open_dataset-9cf3c828-f274-4151-8a35-cc1db931d3d1 Function: execute_task args: ((<function apply at 0x2b47047f9000>, <function open_dataset at 0x2b472e5c2b90>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0027.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,596 - distributed.worker - WARNING - Compute Failed Key: open_dataset-9adb4759-b336-4874-8760-01bd2c74f5a0 Function: execute_task args: ((<function apply at 0x2b47047f9000>, <function open_dataset at 0x2b472e5c2b90>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0077.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,626 - distributed.worker - WARNING - Compute Failed Key: open_dataset-c9d6a61f-51bc-4e02-9df1-526522677194 Function: execute_task args: ((<function apply at 0x2ab55fcc5000>, <function open_dataset at 0x2ab589b115a0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0195.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,634 - distributed.worker - WARNING - Compute Failed Key: open_dataset-16ff4743-d413-4fd1-8f36-8b40a730cec1 Function: execute_task args: ((<function apply at 0x2b0f245b9000>, <function open_dataset at 0x2b0f4e3a8ca0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0333.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,634 - distributed.worker - WARNING - Compute Failed Key: open_dataset-1810866c-da42-468a-9cd7-5471d105ec4c Function: execute_task args: ((<function apply at 0x2b0f245b9000>, <function open_dataset at 0x2b0f4e3a8ca0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0124.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,637 - distributed.worker - WARNING - Compute Failed Key: open_dataset-02aa0b86-f7ed-4dcb-acdf-780c78bc5e68 Function: execute_task args: ((<function apply at 0x2b1b79445000>, <function open_dataset at 0x2b1ba3341240>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0138.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,637 - distributed.worker - WARNING - Compute Failed Key: open_dataset-01595354-4370-412a-a84b-0478aa3caa9d Function: execute_task args: ((<function apply at 0x2b1b79445000>, <function open_dataset at 0x2b1ba3341240>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0308.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:10:27,954 - distributed.worker - WARNING - Compute Failed Key: open_dataset-02d62570-bbd5-429d-a7a0-50820d92ba92 Function: execute_task args: ((<function apply at 0x2b1b79445000>, <function open_dataset at 0x2b1ba3341240>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/07/SEDNA-DELTA_1d_gridT-2D_201307-201307_0525.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'deptht_bounds', 'time_counter_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")'
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) File <timed exec>:6, in <module> File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:681, in datas(catalog_url, dfi, month, year, daskreport, lazy) 676 datadict, paramdict = getdict(dfi) 677 #print('datadict:',datadict) 678 #if datadict == {}: 679 # data=0 680 #else: --> 681 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 682 for s in paramdict: 683 print('param',s,'will be included in data') File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:499, in outputs(catalog_url, datadict, month, year, daskreport, lazy) 496 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"): 497 #ds=load_data_xios_patch(cat,filename,month,catalog_url) 498 print("lazy=",lazy) --> 499 ds = load_data_xios(cat,filename,items,month,year) if ('True' in lazy) else load_data_xios_kerchunk(cat,filename,items,month,year,rome=True) 500 extime=time.time() - start 501 print(' took', extime, 'seconds') File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:374, in load_data_xios(cat, filename, items, month, year) 372 desc=cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).describe() 373 print('using load_data_xios reading ',desc) --> 374 ds = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask() 375 ds = ds[items] 376 return ds File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:69, in DataSourceMixin.to_dask(self) 67 def to_dask(self): 68 """Return xarray object where variables are dask arrays""" ---> 69 return self.read_chunked() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:44, in DataSourceMixin.read_chunked(self) 42 def read_chunked(self): 43 """Return xarray object (which will have chunks)""" ---> 44 self._load_metadata() 45 return self._ds File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake/source/base.py:236, in DataSourceBase._load_metadata(self) 234 """load metadata only if needed""" 235 if self._schema is None: --> 236 self._schema = self._get_schema() 237 self.dtype = self._schema.dtype 238 self.shape = self._schema.shape File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:18, in DataSourceMixin._get_schema(self) 15 self.urlpath = self._get_cache(self.urlpath)[0] 17 if self._ds is None: ---> 18 self._open_dataset() 20 metadata = { 21 'dims': dict(self._ds.dims), 22 'data_vars': {k: list(self._ds[k].coords) 23 for k in self._ds.data_vars.keys()}, 24 'coords': tuple(self._ds.coords.keys()), 25 } 26 if getattr(self, 'on_server', False): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/netcdf.py:92, in NetCDFSource._open_dataset(self) 88 else: 89 # https://github.com/intake/filesystem_spec/issues/476#issuecomment-732372918 90 url = fsspec.open(self.urlpath, **self.storage_options).open() ---> 92 self._ds = _open_dataset(url, chunks=self.chunks, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:980, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs) 975 datasets = [preprocess(ds) for ds in datasets] 977 if parallel: 978 # calling compute here will return the datasets/file_objs lists, 979 # the underlying datasets will still be stored as dask arrays --> 980 datasets, closers = dask.compute(datasets, closers) 982 # Combine all datasets, closing them in case of a ValueError 983 try: File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/base.py:598, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs) 595 keys.append(x.__dask_keys__()) 596 postcomputes.append(x.__dask_postcompute__()) --> 598 results = schedule(dsk, keys, **kwargs) 599 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:3001, in Client.get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2999 should_rejoin = False 3000 try: -> 3001 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 3002 finally: 3003 for f in futures.values(): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2175, in Client.gather(self, futures, errors, direct, asynchronous) 2173 else: 2174 local_worker = None -> 2175 return self.sync( 2176 self._gather, 2177 futures, 2178 errors=errors, 2179 direct=direct, 2180 local_worker=local_worker, 2181 asynchronous=asynchronous, 2182 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:338, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 336 return future 337 else: --> 338 return sync( 339 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 340 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:405, in sync(loop, func, callback_timeout, *args, **kwargs) 403 if error: 404 typ, exc, tb = error --> 405 raise exc.with_traceback(tb) 406 else: 407 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:378, in sync.<locals>.f() 376 future = asyncio.wait_for(future, callback_timeout) 377 future = asyncio.ensure_future(future) --> 378 result = yield future 379 except Exception: 380 error = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/tornado/gen.py:762, in Runner.run(self) 759 exc_info = None 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2038, in Client._gather(self, futures, errors, direct, local_worker) 2036 exc = CancelledError(key) 2037 else: -> 2038 raise exception.with_traceback(traceback) 2039 raise exc 2040 if errors == "skip": File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/utils.py:41, in apply() 39 def apply(func, args, kwargs=None): 40 if kwargs: ---> 41 return func(*args, **kwargs) 42 else: 43 return func(*args) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:515, in open_dataset() 512 kwargs.update(backend_kwargs) 514 if engine is None: --> 515 engine = plugins.guess_engine(filename_or_obj) 517 backend = plugins.get_backend(engine) 519 decoders = _resolve_decoders_kwargs( 520 decode_cf, 521 open_backend_dataset_parameters=backend.open_dataset_parameters, (...) 527 decode_coords=decode_coords, 528 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/plugins.py:155, in guess_engine() 147 else: 148 error_msg = ( 149 "found the following matches with the input file in xarray's IO " 150 f"backends: {compatible_engines}. But their dependencies may not be installed, see:\n" 151 "https://docs.xarray.dev/en/stable/user-guide/io.html \n" 152 "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html" 153 ) --> 155 raise ValueError(error_msg) ValueError: did not find a match in any of xarray's currently installed IO backends ['netcdf4', 'scipy', 'zarr']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see: https://docs.xarray.dev/en/stable/getting-started-guide/installing.html https://docs.xarray.dev/en/stable/user-guide/io.html
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) File <timed eval>:1, in <module> NameError: name 'data' is not defined