%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/monitor.py'>
If you submit the job with job scheduler; below are list of enviroment variable one can pass
local : if True run dask local cluster, if not true, put number of workers setted in the 'local' if no 'local ' given, local will be setted automatically to 'True'
%env ychunk='2', #%env tchunk='2'
controls chunk. 'False' sets no modification from original netcdf file's chunk.
ychunk=10 will group the original netcdf file to 10 by 10
tchunk=1 will chunk the time coordinate one by one
%env file_exp=
'file_exp': Which 'experiment' name is it? this corresopnds to intake catalog name without path and .yaml
#%env year=
for Validation, this correspoinds to path/year/month 's year for monitoring, this corresponids to 'date' having means do all files in the monitoring directory setting it as 0[0-9] &1[0-9]& [2-3][0-9], the job can be separated in three lots. For DELTA experiment, year corresponds to really 'year'
%env month=
for monitoring this corresponds to file path path-XIOS.{month}/
For DELTA experiment, year corresponds to really 'month'
proceed saving? True or False , Default is setted as True
proceed plotting? True or False , Default is setted as True
proceed computation? or just load computed result? True or False , Default is setted as True
save output file used for plotting
using kerchunked file -> False, not using kerhcunk -> True
name of control file to be used for computation/plots/save/ We have number of M_xxx.csv
Monitor.sh calls M_MLD_2D
and AWTD.sh, Fluxnet.sh, Siconc.sh, IceClim.sh, FWC_SSH.sh, Integrals.sh , Sections.sh
M_AWTMD
M_Fluxnet
M_Ice_quantities
M_IceClim M_IceConce M_IceThick
M_FWC_2D M_FWC_integrals M_FWC_SSH M_SSH_anomaly
M_Mean_temp_velo M_Mooring
M_Sectionx M_Sectiony
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene8002.c-irene.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False tgcc local cluster starting This code is running on irene8002.c-irene.tgcc.ccc.cea.fr using SEDNA_DELTA_MONITOR file experiment, read from ../lib/SEDNA_DELTA_MONITOR.yaml on year= 2013 on month= 06 outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6570279irene8002.c-irene.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_06M_SSH_anomaly/ CPU times: user 537 ms, sys: 166 ms, total: 703 ms Wall time: 15 s
Client-ab454acb-2487-11ed-a914-080038bfdcae
Connection method: Cluster object | Cluster type: distributed.LocalCluster |
Dashboard: http://127.0.0.1:8787/status |
f173ba8f
Dashboard: http://127.0.0.1:8787/status | Workers: 12 |
Total threads: 48 | Total memory: 2.86 TiB |
Status: running | Using processes: True |
Scheduler-c2f51284-7819-4812-9469-0cabf7a8fe91
Comm: tcp://127.0.0.1:35664 | Workers: 12 |
Dashboard: http://127.0.0.1:8787/status | Total threads: 48 |
Started: Just now | Total memory: 2.86 TiB |
Comm: tcp://127.0.0.1:35650 | Total threads: 4 |
Dashboard: http://127.0.0.1:43113/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:37158 | |
Local directory: /tmp/dask-worker-space/worker-z8tq2mlg |
Comm: tcp://127.0.0.1:41012 | Total threads: 4 |
Dashboard: http://127.0.0.1:34127/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:34616 | |
Local directory: /tmp/dask-worker-space/worker-b1qj0tn7 |
Comm: tcp://127.0.0.1:45117 | Total threads: 4 |
Dashboard: http://127.0.0.1:33291/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:46392 | |
Local directory: /tmp/dask-worker-space/worker-b9hgyhr_ |
Comm: tcp://127.0.0.1:46603 | Total threads: 4 |
Dashboard: http://127.0.0.1:36561/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:38839 | |
Local directory: /tmp/dask-worker-space/worker-7buba0fq |
Comm: tcp://127.0.0.1:37654 | Total threads: 4 |
Dashboard: http://127.0.0.1:37023/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:35931 | |
Local directory: /tmp/dask-worker-space/worker-c1ya05q4 |
Comm: tcp://127.0.0.1:45820 | Total threads: 4 |
Dashboard: http://127.0.0.1:43157/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:36336 | |
Local directory: /tmp/dask-worker-space/worker-rlf2wvf1 |
Comm: tcp://127.0.0.1:40660 | Total threads: 4 |
Dashboard: http://127.0.0.1:39752/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:36397 | |
Local directory: /tmp/dask-worker-space/worker-mw8uak6q |
Comm: tcp://127.0.0.1:44718 | Total threads: 4 |
Dashboard: http://127.0.0.1:34114/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:37803 | |
Local directory: /tmp/dask-worker-space/worker-gfdecb1q |
Comm: tcp://127.0.0.1:33968 | Total threads: 4 |
Dashboard: http://127.0.0.1:43039/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:45840 | |
Local directory: /tmp/dask-worker-space/worker-0e80xyvd |
Comm: tcp://127.0.0.1:35358 | Total threads: 4 |
Dashboard: http://127.0.0.1:45873/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:45079 | |
Local directory: /tmp/dask-worker-space/worker-h_fnwf0u |
Comm: tcp://127.0.0.1:32833 | Total threads: 4 |
Dashboard: http://127.0.0.1:45522/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:46435 | |
Local directory: /tmp/dask-worker-space/worker-9jv0myfp |
Comm: tcp://127.0.0.1:44239 | Total threads: 4 |
Dashboard: http://127.0.0.1:37318/status | Memory: 244.27 GiB |
Nanny: tcp://127.0.0.1:42409 | |
Local directory: /tmp/dask-worker-space/worker-obvrjia7 |
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
SSH_anomaly | gridT-2D.ssh | calc.SSH_anomaly(data) | ALL | maps | Spectral_r | (-0.35,0.35) | m | M-2 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True lazy= True ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: nested concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}} 0 read gridT-2D ['ssh'] lazy= True using load_data_xios reading gridT-2D using load_data_xios reading {'name': 'data_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO outputs from different xios server', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'name of config', 'type': 'str', 'default': '/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d'}, {'name': 'fileexp', 'description': 'name of config', 'type': 'str', 'default': 'SEDNA-DELTA'}, {'name': 'month', 'description': 'running number 2 digit', 'type': 'str', 'default': '02'}, {'name': 'freq', 'description': '1d or 1m', 'type': 'str', 'default': '1d'}, {'name': 'year', 'description': 'last digits of yearmonthdate.', 'type': 'str', 'default': '2012'}, {'name': 'file', 'description': 'file name', 'type': 'str', 'default': 'icemod'}, {'name': 'eio', 'description': 'xios mpi number', 'type': 'str', 'default': '0[0-5][0-9][0-9]'}], 'metadata': {}, 'args': {'urlpath': '{{path}}/{{year}}/{{month}}/*{{file}}_*_{{eio}}.nc', 'combine': 'nested', 'concat_dim': 'y'}}
HDF5-DIAG: Error detected in HDF5 (1.12.1) HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 1thread 1: : #000: H5F.c line 620 in H5Fopen(): unable to open file #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility major: File accessibility minor: Unable to open file minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer major: Virtual Object Layer minor: Iteration failed minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library major: Plugin for dynamically loaded library minor: Iteration failed minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library major: Plugin for dynamically loaded library minor: Can't open directory or file minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer major: Virtual Object Layer minor: Can't open object minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility major: File accessibility minor: Unable to open file minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility major: File accessibility minor: Read failed minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 638976, sblock->base_addr = 0, stored_eof = 22816004 #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 3522560, sblock->base_addr = 0, stored_eof = 19835133 major: File accessibility major: File accessibility minor: File has been truncated minor: File has been truncated HDF5-DIAG: Error detected in HDF5 (1.12.1) thread 2: #000: H5F.c line 620 in H5Fopen(): unable to open file major: File accessibility minor: Unable to open file #001: H5VLcallback.c line 3501 in H5VL_file_open(): failed to iterate over available VOL connector plugins major: Virtual Object Layer minor: Iteration failed #002: H5PLpath.c line 578 in H5PL__path_table_iterate(): can't iterate over plugins in plugin path '(null)' major: Plugin for dynamically loaded library minor: Iteration failed #003: H5PLpath.c line 620 in H5PL__path_table_iterate_process_path(): can't open directory: /ccc/cont003/home/ra5563/ra5563/monitor/lib/hdf5/plugin major: Plugin for dynamically loaded library minor: Can't open directory or file #004: H5VLcallback.c line 3351 in H5VL__file_open(): open failed major: Virtual Object Layer minor: Can't open object #005: H5VLnative_file.c line 97 in H5VL__native_file_open(): unable to open file major: File accessibility minor: Unable to open file #006: H5Fint.c line 1990 in H5F_open(): unable to read superblock major: File accessibility minor: Read failed #007: H5Fsuper.c line 614 in H5F__super_read(): truncated file: eof = 4038656, sblock->base_addr = 0, stored_eof = 24434996 major: File accessibility minor: File has been truncated 2022-08-25 17:08:05,146 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b22cdd54-83ef-4249-88ed-c00f9e76dbc1 Function: execute_task args: ((<function apply at 0x2b8ca9275000>, <function open_dataset at 0x2b8cd3101630>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0400.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:05,147 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2eb0df3a-85dc-4409-afd8-7f0219bfabfe Function: execute_task args: ((<function apply at 0x2af1981e5000>, <function open_dataset at 0x2af1c2055a20>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0447.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:05,182 - distributed.worker - WARNING - Compute Failed Key: open_dataset-01d39315-c9e1-4b33-8688-80d0dbb15d70 Function: execute_task args: ((<function apply at 0x2b84fda55000>, <function open_dataset at 0x2b852389fb50>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0257.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:05,256 - distributed.worker - WARNING - Compute Failed Key: open_dataset-88c1d60f-bd4a-4fe2-82b1-bf529bdd7176 Function: execute_task args: ((<function apply at 0x2add71f89000>, <function open_dataset at 0x2add97dbb250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0526.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:08:05,541 - distributed.worker - WARNING - Compute Failed Key: open_dataset-5d31b7d3-8626-47d5-93b8-fe9ac3acc168 Function: execute_task args: ((<function apply at 0x2ae62a5e9000>, <function open_dataset at 0x2ae6544130a0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0379.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:05,541 - distributed.worker - WARNING - Compute Failed Key: open_dataset-1b10c2b2-c591-4503-9bdb-3d84127fbde7 Function: execute_task args: ((<function apply at 0x2ba60e8f9000>, <function open_dataset at 0x2ba63873d480>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0397.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:05,542 - distributed.worker - WARNING - Compute Failed Key: open_dataset-00fb84c0-091c-497b-bd68-49b8f8bfc14d Function: execute_task args: ((<function apply at 0x2b84fda55000>, <function open_dataset at 0x2b852389fb50>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0387.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:08:05,640 - distributed.worker - WARNING - Compute Failed Key: open_dataset-87ce13c4-1241-47ab-8bcb-143619fca279 Function: execute_task args: ((<function apply at 0x2add71f89000>, <function open_dataset at 0x2add97dbb250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0287.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:08:05,641 - distributed.worker - WARNING - Compute Failed Key: open_dataset-85ffa4ea-0b47-478a-9940-8bd2ab7e40ea Function: execute_task args: ((<function apply at 0x2add71f89000>, <function open_dataset at 0x2add97dbb250>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0029.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:08:20,090 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b1ade3cd-9f7c-405c-8ddb-831a10db95ba Function: execute_task args: ((<function apply at 0x2b8ca9275000>, <function open_dataset at 0x2b8cd3101630>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0323.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:08:20,091 - distributed.worker - WARNING - Compute Failed Key: open_dataset-b1fa28ca-3f55-4250-9b7f-1b7e6583ac3b Function: execute_task args: ((<function apply at 0x2b8ca9275000>, <function open_dataset at 0x2b8cd3101630>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0534.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:20,104 - distributed.worker - WARNING - Compute Failed Key: open_dataset-ea3dfa0c-bde6-4353-acea-7525e4ccb46c Function: execute_task args: ((<function apply at 0x2ac261831000>, <function open_dataset at 0x2ac28b6535b0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0177.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: "OSError(-101, 'NetCDF: HDF error')" 2022-08-25 17:08:20,108 - distributed.worker - WARNING - Compute Failed Key: open_dataset-c6528a76-baeb-4ea1-9424-a22037354a01 Function: execute_task args: ((<function apply at 0x2b2a6b235000>, <function open_dataset at 0x2b2a94fe1c60>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0179.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:20,108 - distributed.worker - WARNING - Compute Failed Key: open_dataset-c621587a-fb0f-4f72-97b5-c3914d32d07f Function: execute_task args: ((<function apply at 0x2b2a6b235000>, <function open_dataset at 0x2b2a94fe1c60>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0381.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:20,115 - distributed.worker - WARNING - Compute Failed Key: open_dataset-2e1ec910-a697-4af4-84de-6c77ff8d34ba Function: execute_task args: ((<function apply at 0x2af1981e5000>, <function open_dataset at 0x2af1c2055a20>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0390.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:20,129 - distributed.worker - WARNING - Compute Failed Key: open_dataset-7034dcf2-4ff2-4eec-921a-e06e9d527c23 Function: execute_task args: ((<function apply at 0x2b70e4785000>, <function open_dataset at 0x2b710e579fc0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0258.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")' 2022-08-25 17:08:20,129 - distributed.worker - WARNING - Compute Failed Key: open_dataset-7104f2ee-1ef1-46ad-a2f2-c57ac2bfe72b Function: execute_task args: ((<function apply at 0x2b70e4785000>, <function open_dataset at 0x2b710e579fc0>, ['/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2013/06/SEDNA-DELTA_1d_gridT-2D_201306-201306_0393.nc'], (<class 'dict'>, [['engine', None], ['chunks', (<class 'dict'>, [])], ['drop_variables', (<class 'set'>, ['time_centered_bounds', 'time_counter_bounds', 'deptht_bounds', 'depthu_bounds'])]]))) kwargs: {} Exception: 'ValueError("did not find a match in any of xarray\'s currently installed IO backends [\'netcdf4\', \'scipy\', \'zarr\']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:\\nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\\nhttps://docs.xarray.dev/en/stable/user-guide/io.html")'
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) File <timed exec>:6, in <module> File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:681, in datas(catalog_url, dfi, month, year, daskreport, lazy) 676 datadict, paramdict = getdict(dfi) 677 #print('datadict:',datadict) 678 #if datadict == {}: 679 # data=0 680 #else: --> 681 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 682 for s in paramdict: 683 print('param',s,'will be included in data') File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:499, in outputs(catalog_url, datadict, month, year, daskreport, lazy) 496 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"): 497 #ds=load_data_xios_patch(cat,filename,month,catalog_url) 498 print("lazy=",lazy) --> 499 ds = load_data_xios(cat,filename,items,month,year) if ('True' in lazy) else load_data_xios_kerchunk(cat,filename,items,month,year,rome=True) 500 extime=time.time() - start 501 print(' took', extime, 'seconds') File /ccc/work/cont003/gen7420/talandel/TOOLS/monitor-sedna/notebook/core/load.py:374, in load_data_xios(cat, filename, items, month, year) 372 desc=cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).describe() 373 print('using load_data_xios reading ',desc) --> 374 ds = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask() 375 ds = ds[items] 376 return ds File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:69, in DataSourceMixin.to_dask(self) 67 def to_dask(self): 68 """Return xarray object where variables are dask arrays""" ---> 69 return self.read_chunked() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:44, in DataSourceMixin.read_chunked(self) 42 def read_chunked(self): 43 """Return xarray object (which will have chunks)""" ---> 44 self._load_metadata() 45 return self._ds File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake/source/base.py:236, in DataSourceBase._load_metadata(self) 234 """load metadata only if needed""" 235 if self._schema is None: --> 236 self._schema = self._get_schema() 237 self.dtype = self._schema.dtype 238 self.shape = self._schema.shape File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:18, in DataSourceMixin._get_schema(self) 15 self.urlpath = self._get_cache(self.urlpath)[0] 17 if self._ds is None: ---> 18 self._open_dataset() 20 metadata = { 21 'dims': dict(self._ds.dims), 22 'data_vars': {k: list(self._ds[k].coords) 23 for k in self._ds.data_vars.keys()}, 24 'coords': tuple(self._ds.coords.keys()), 25 } 26 if getattr(self, 'on_server', False): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/netcdf.py:92, in NetCDFSource._open_dataset(self) 88 else: 89 # https://github.com/intake/filesystem_spec/issues/476#issuecomment-732372918 90 url = fsspec.open(self.urlpath, **self.storage_options).open() ---> 92 self._ds = _open_dataset(url, chunks=self.chunks, **kwargs) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:980, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs) 975 datasets = [preprocess(ds) for ds in datasets] 977 if parallel: 978 # calling compute here will return the datasets/file_objs lists, 979 # the underlying datasets will still be stored as dask arrays --> 980 datasets, closers = dask.compute(datasets, closers) 982 # Combine all datasets, closing them in case of a ValueError 983 try: File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/base.py:598, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs) 595 keys.append(x.__dask_keys__()) 596 postcomputes.append(x.__dask_postcompute__()) --> 598 results = schedule(dsk, keys, **kwargs) 599 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:3001, in Client.get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2999 should_rejoin = False 3000 try: -> 3001 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 3002 finally: 3003 for f in futures.values(): File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2175, in Client.gather(self, futures, errors, direct, asynchronous) 2173 else: 2174 local_worker = None -> 2175 return self.sync( 2176 self._gather, 2177 futures, 2178 errors=errors, 2179 direct=direct, 2180 local_worker=local_worker, 2181 asynchronous=asynchronous, 2182 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:338, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 336 return future 337 else: --> 338 return sync( 339 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 340 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:405, in sync(loop, func, callback_timeout, *args, **kwargs) 403 if error: 404 typ, exc, tb = error --> 405 raise exc.with_traceback(tb) 406 else: 407 return result File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/utils.py:378, in sync.<locals>.f() 376 future = asyncio.wait_for(future, callback_timeout) 377 future = asyncio.ensure_future(future) --> 378 result = yield future 379 except Exception: 380 error = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/tornado/gen.py:762, in Runner.run(self) 759 exc_info = None 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info() File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/distributed/client.py:2038, in Client._gather(self, futures, errors, direct, local_worker) 2036 exc = CancelledError(key) 2037 else: -> 2038 raise exception.with_traceback(traceback) 2039 raise exc 2040 if errors == "skip": File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/dask/utils.py:41, in apply() 39 def apply(func, args, kwargs=None): 40 if kwargs: ---> 41 return func(*args, **kwargs) 42 else: 43 return func(*args) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:515, in open_dataset() 512 kwargs.update(backend_kwargs) 514 if engine is None: --> 515 engine = plugins.guess_engine(filename_or_obj) 517 backend = plugins.get_backend(engine) 519 decoders = _resolve_decoders_kwargs( 520 decode_cf, 521 open_backend_dataset_parameters=backend.open_dataset_parameters, (...) 527 decode_coords=decode_coords, 528 ) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/plugins.py:155, in guess_engine() 147 else: 148 error_msg = ( 149 "found the following matches with the input file in xarray's IO " 150 f"backends: {compatible_engines}. But their dependencies may not be installed, see:\n" 151 "https://docs.xarray.dev/en/stable/user-guide/io.html \n" 152 "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html" 153 ) --> 155 raise ValueError(error_msg) ValueError: did not find a match in any of xarray's currently installed IO backends ['netcdf4', 'scipy', 'zarr']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see: https://docs.xarray.dev/en/stable/getting-started-guide/installing.html https://docs.xarray.dev/en/stable/user-guide/io.html
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
--------------------------------------------------------------------------- NameError Traceback (most recent call last) File <timed eval>:1, in <module> NameError: name 'data' is not defined