
%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()

from core import  load, zoom, calc, save,plots,monitor


#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)

<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>


%%time
# 'savefig': Do we save output in html? or not. keep it true. 
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client

local True
using host= irene4722.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16
10000000000
False
tgcc local cluster starting
This code is running on  irene4722.c-irene.mg1.tgcc.ccc.cea.fr using  SEDNA_DELTA_MONITOR file experiment, read from  ../lib/SEDNA_DELTA_MONITOR.yaml  on year= 2012  on month= 01  outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6419176irene4722.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_01M_SSH_anomaly/
CPU times: user 3.91 s, sys: 751 ms, total: 4.66 s
Wall time: 1min 40s


df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df


%%time
import os
calcswitch=os.environ.get('calc', 'True') 
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any()) 
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0 
data

calcswitch= True df.Inputs != nothing True lazy= False
../lib/SEDNA_DELTA_MONITOR.yaml
using param_xios reading  ../lib/SEDNA_DELTA_MONITOR.yaml
using param_xios reading  <bound method DataSourceBase.describe of sources:
  param_xios:
    args:
      combine: nested
      concat_dim: y
      urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc
      xarray_kwargs:
        compat: override
        coords: minimal
        data_vars: minimal
        parallel: true
    description: SEDNA NEMO parameters from MPI output  nav_lon lat fails
    driver: intake_xarray.netcdf.NetCDFSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
{'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output  nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}}
0 read gridT-2D ['ssh']
lazy= False
using load_data_xios_kerchunk reading  gridT-2D
using load_data_xios_kerchunk reading  <bound method DataSourceBase.describe of sources:
  data_xios_kerchunk:
    args:
      consolidated: false
      storage_options:
        fo: file:////ccc/cont003/home/ra5563/ra5563/catalogue/DELTA/201201/gridT-2D_0[0-5][0-9][0-9].json
        target_protocol: file
      urlpath: reference://
    description: CREG025 NEMO outputs from different xios server in kerchunk format
    driver: intake_xarray.xzarr.ZarrSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
File <timed exec>:6, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py:677, in datas(catalog_url, dfi, month, year, daskreport, lazy)
    672 datadict, paramdict = getdict(dfi)
    673 #print('datadict:',datadict)
    674 #if datadict == {}:
    675 #    data=0
    676 #else:
--> 677 data=outputs(catalog_url,datadict,month,year,daskreport,lazy) 
    678 for s in paramdict:
    679     print('param',s,'will be included in data')

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py:496, in outputs(catalog_url, datadict, month, year, daskreport, lazy)
    493 with performance_report(filename=daskreport+"_load_output_"+filename+"_"+month+year+".html"):
    494     #ds=load_data_xios_patch(cat,filename,month,catalog_url) 
    495     print("lazy=",lazy)
--> 496     ds = load_data_xios(cat,filename,items,month,year) if ('True' in lazy)  else load_data_xios_kerchunk(cat,filename,items,month,year,rome=True)
    497 extime=time.time() - start
    498 print('      took', extime, 'seconds')

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py:467, in load_data_xios_kerchunk(cat, filename, items, month, year, rome)
    465 desc=cat.data_xios_kerchunk(file=filename,month=month,year=year).describe         
    466 print('using load_data_xios_kerchunk reading ',desc)
--> 467 ds_x= [ prep(
    468     cat.data_xios_kerchunk(
    469         file=filename,month=month,year=year,eio=f'{xios:04}' 
    470     ).to_dask().drop_vars(dro,errors='ignore')[items])
    471        for xios in xioss]
    473 return xr.concat(ds_x,dim='y',compat="override",coords="minimal")

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/load.py:470, in <listcomp>(.0)
    465 desc=cat.data_xios_kerchunk(file=filename,month=month,year=year).describe         
    466 print('using load_data_xios_kerchunk reading ',desc)
    467 ds_x= [ prep(
    468     cat.data_xios_kerchunk(
    469         file=filename,month=month,year=year,eio=f'{xios:04}' 
--> 470     ).to_dask().drop_vars(dro,errors='ignore')[items])
    471        for xios in xioss]
    473 return xr.concat(ds_x,dim='y',compat="override",coords="minimal")

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:69, in DataSourceMixin.to_dask(self)
     67 def to_dask(self):
     68     """Return xarray object where variables are dask arrays"""
---> 69     return self.read_chunked()

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:44, in DataSourceMixin.read_chunked(self)
     42 def read_chunked(self):
     43     """Return xarray object (which will have chunks)"""
---> 44     self._load_metadata()
     45     return self._ds

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake/source/base.py:236, in DataSourceBase._load_metadata(self)
    234 """load metadata only if needed"""
    235 if self._schema is None:
--> 236     self._schema = self._get_schema()
    237     self.dtype = self._schema.dtype
    238     self.shape = self._schema.shape

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/base.py:18, in DataSourceMixin._get_schema(self)
     15 self.urlpath = self._get_cache(self.urlpath)[0]
     17 if self._ds is None:
---> 18     self._open_dataset()
     20     metadata = {
     21         'dims': dict(self._ds.dims),
     22         'data_vars': {k: list(self._ds[k].coords)
     23                       for k in self._ds.data_vars.keys()},
     24         'coords': tuple(self._ds.coords.keys()),
     25     }
     26     if getattr(self, 'on_server', False):

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/intake_xarray/xzarr.py:46, in ZarrSource._open_dataset(self)
     44     self._ds = xr.open_mfdataset(self.urlpath, **kw)
     45 else:
---> 46     self._ds = xr.open_dataset(self.urlpath, **kw)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:531, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
    519 decoders = _resolve_decoders_kwargs(
    520     decode_cf,
    521     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)
    527     decode_coords=decode_coords,
    528 )
    530 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 531 backend_ds = backend.open_dataset(
    532     filename_or_obj,
    533     drop_variables=drop_variables,
    534     **decoders,
    535     **kwargs,
    536 )
    537 ds = _dataset_from_backend_dataset(
    538     backend_ds,
    539     filename_or_obj,
   (...)
    547     **kwargs,
    548 )
    549 return ds

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/zarr.py:837, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel)
    817 def open_dataset(
    818     self,
    819     filename_or_obj,
   (...)
    833     stacklevel=3,
    834 ):
    836     filename_or_obj = _normalize_path(filename_or_obj)
--> 837     store = ZarrStore.open_group(
    838         filename_or_obj,
    839         group=group,
    840         mode=mode,
    841         synchronizer=synchronizer,
    842         consolidated=consolidated,
    843         consolidate_on_close=False,
    844         chunk_store=chunk_store,
    845         storage_options=storage_options,
    846         stacklevel=stacklevel + 1,
    847     )
    849     store_entrypoint = StoreBackendEntrypoint()
    850     with close_on_error(store):

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/zarr.py:406, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, stacklevel)
    404     zarr_group = zarr.open_consolidated(store, **open_kwargs)
    405 else:
--> 406     zarr_group = zarr.open_group(store, **open_kwargs)
    407 return cls(
    408     zarr_group,
    409     mode,
   (...)
    413     safe_chunks,
    414 )

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/zarr/hierarchy.py:1316, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version)
   1270 """Open a group using file-mode-like semantics.
   1271 
   1272 Parameters
   (...)
   1312 
   1313 """
   1315 # handle polymorphic store arg
-> 1316 store = _normalize_store_arg(
   1317     store, storage_options=storage_options, mode=mode,
   1318     zarr_version=zarr_version)
   1319 if zarr_version is None:
   1320     zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/zarr/hierarchy.py:1192, in _normalize_store_arg(store, storage_options, mode, zarr_version)
   1190 if store is None:
   1191     return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
-> 1192 return normalize_store_arg(store,
   1193                            storage_options=storage_options, mode=mode,
   1194                            zarr_version=zarr_version)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/zarr/storage.py:170, in normalize_store_arg(store, storage_options, mode, zarr_version)
    168     from zarr._storage.v3 import _normalize_store_arg_v3
    169     normalize_store = _normalize_store_arg_v3
--> 170 return normalize_store(store, storage_options, mode)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/zarr/storage.py:143, in _normalize_store_arg_v2(store, storage_options, mode)
    141 if isinstance(store, str):
    142     if "://" in store or "::" in store:
--> 143         return FSStore(store, mode=mode, **(storage_options or {}))
    144     elif storage_options:
    145         raise ValueError("storage_options passed with non-fsspec path")

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/zarr/storage.py:1321, in FSStore.__init__(self, url, normalize_keys, key_separator, mode, exceptions, dimension_separator, fs, check, create, missing_exceptions, **storage_options)
   1319 if protocol in (None, "file") and not storage_options.get("auto_mkdir"):
   1320     storage_options["auto_mkdir"] = True
-> 1321 self.map = fsspec.get_mapper(url, **{**mapper_options, **storage_options})
   1322 self.fs = self.map.fs  # for direct operations
   1323 self.path = self.fs._strip_protocol(url)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/mapping.py:230, in get_mapper(url, check, create, missing_exceptions, alternate_root, **kwargs)
    199 """Create key-value interface for given URL and options
    200 
    201 The URL will be of the form "protocol://location" and point to the root
   (...)
    227 ``FSMap`` instance, the dict-like key-value store.
    228 """
    229 # Removing protocol here - could defer to each open() on the backend
--> 230 fs, urlpath = url_to_fs(url, **kwargs)
    231 root = alternate_root if alternate_root is not None else urlpath
    232 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/core.py:412, in url_to_fs(url, **kwargs)
    410     options = cls._get_kwargs_from_urls(url)
    411     update_storage_options(options, kwargs)
--> 412     fs = cls(**options)
    413     urlpath = fs._strip_protocol(url)
    414 return fs, urlpath

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/spec.py:76, in _Cached.__call__(cls, *args, **kwargs)
     74     return cls._cache[token]
     75 else:
---> 76     obj = super().__call__(*args, **kwargs)
     77     # Setting _fs_token here causes some static linters to complain.
     78     obj._fs_token_ = token

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/implementations/reference.py:140, in ReferenceFileSystem.__init__(self, fo, target, ref_storage_args, target_protocol, target_options, remote_protocol, remote_options, fs, template_overrides, simple_templates, loop, **kwargs)
    138 dic = dict(**(ref_storage_args or target_options or {}), **extra)
    139 # text JSON
--> 140 with open(fo, "rb", **dic) as f:
    141     logger.info("Read reference from URL %s", fo)
    142     text = f.read()

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/core.py:104, in OpenFile.__enter__(self)
    101 def __enter__(self):
    102     mode = self.mode.replace("t", "").replace("b", "") + "b"
--> 104     f = self.fs.open(self.path, mode=mode)
    106     self.fobjects = [f]
    108     if self.compression is not None:

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/spec.py:1037, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
   1035 else:
   1036     ac = kwargs.pop("autocommit", not self._intrans)
-> 1037     f = self._open(
   1038         path,
   1039         mode=mode,
   1040         block_size=block_size,
   1041         autocommit=ac,
   1042         cache_options=cache_options,
   1043         **kwargs,
   1044     )
   1045     if compression is not None:
   1046         from fsspec.compression import compr

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/implementations/local.py:159, in LocalFileSystem._open(self, path, mode, block_size, **kwargs)
    157 if self.auto_mkdir and "w" in mode:
    158     self.makedirs(self._parent(path), exist_ok=True)
--> 159 return LocalFileOpener(path, mode, fs=self, **kwargs)

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/implementations/local.py:254, in LocalFileOpener.__init__(self, path, mode, autocommit, fs, compression, **kwargs)
    252 self.compression = get_compression(path, compression)
    253 self.blocksize = io.DEFAULT_BUFFER_SIZE
--> 254 self._open()

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/fsspec/implementations/local.py:259, in LocalFileOpener._open(self)
    257 if self.f is None or self.f.closed:
    258     if self.autocommit or "w" not in self.mode:
--> 259         self.f = open(self.path, mode=self.mode)
    260         if self.compression:
    261             compress = compr[self.compression]

FileNotFoundError: [Errno 2] No such file or directory: '//ccc/cont003/home/ra5563/ra5563/catalogue/DELTA/201201/gridT-2D_0000.json'


%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
            )

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
File <timed eval>:1, in <module>

NameError: name 'data' is not defined

Dashboard: http://127.0.0.1:8787/status	Workers: 64
Total threads: 256	Total memory: 251.06 GiB
Status: running	Using processes: True

Comm: tcp://127.0.0.1:33069	Workers: 64
Dashboard: http://127.0.0.1:8787/status	Total threads: 256
Started: 1 minute ago	Total memory: 251.06 GiB

Comm: tcp://127.0.0.1:33451	Total threads: 4
Dashboard: http://127.0.0.1:39103/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:38915
Local directory: /tmp/dask-worker-space/worker-g5t874vr

Comm: tcp://127.0.0.1:38505	Total threads: 4
Dashboard: http://127.0.0.1:36598/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:42111
Local directory: /tmp/dask-worker-space/worker-9_uo5m02

Comm: tcp://127.0.0.1:34932	Total threads: 4
Dashboard: http://127.0.0.1:42087/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:45747
Local directory: /tmp/dask-worker-space/worker-k45zggwh

Connection method: Cluster object	Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Comm: tcp://127.0.0.1:41861	Total threads: 4
Dashboard: http://127.0.0.1:44058/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:45442
Local directory: /tmp/dask-worker-space/worker-8rgt9jl3

Comm: tcp://127.0.0.1:45615	Total threads: 4
Dashboard: http://127.0.0.1:43688/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:41945
Local directory: /tmp/dask-worker-space/worker-lpam6fhu

Comm: tcp://127.0.0.1:40460	Total threads: 4
Dashboard: http://127.0.0.1:34037/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:36691
Local directory: /tmp/dask-worker-space/worker-quvcb7ig

Comm: tcp://127.0.0.1:39991	Total threads: 4
Dashboard: http://127.0.0.1:39990/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:36961
Local directory: /tmp/dask-worker-space/worker-mikdhuww

Comm: tcp://127.0.0.1:42001	Total threads: 4
Dashboard: http://127.0.0.1:45450/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:37633
Local directory: /tmp/dask-worker-space/worker-rkj9yjb5

Comm: tcp://127.0.0.1:35151	Total threads: 4
Dashboard: http://127.0.0.1:43863/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:42978
Local directory: /tmp/dask-worker-space/worker-x1bkjnnn

Comm: tcp://127.0.0.1:46602	Total threads: 4
Dashboard: http://127.0.0.1:43745/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:38796
Local directory: /tmp/dask-worker-space/worker-m0kq98k7

Comm: tcp://127.0.0.1:34874	Total threads: 4
Dashboard: http://127.0.0.1:46326/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:34258
Local directory: /tmp/dask-worker-space/worker-4b36_253

Comm: tcp://127.0.0.1:43447	Total threads: 4
Dashboard: http://127.0.0.1:46532/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:41132
Local directory: /tmp/dask-worker-space/worker-yzdykm48

Comm: tcp://127.0.0.1:43467	Total threads: 4
Dashboard: http://127.0.0.1:40117/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40171
Local directory: /tmp/dask-worker-space/worker-31nmglmo

Comm: tcp://127.0.0.1:39169	Total threads: 4
Dashboard: http://127.0.0.1:38131/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:37991
Local directory: /tmp/dask-worker-space/worker-9tzo3ymv

Comm: tcp://127.0.0.1:37128	Total threads: 4
Dashboard: http://127.0.0.1:39744/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:44801
Local directory: /tmp/dask-worker-space/worker-9trrma8z

Comm: tcp://127.0.0.1:45157	Total threads: 4
Dashboard: http://127.0.0.1:36145/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:44854
Local directory: /tmp/dask-worker-space/worker-frs_8tlq

Comm: tcp://127.0.0.1:46000	Total threads: 4
Dashboard: http://127.0.0.1:43667/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40152
Local directory: /tmp/dask-worker-space/worker-bev6d5y8

Comm: tcp://127.0.0.1:34418	Total threads: 4
Dashboard: http://127.0.0.1:36311/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40317
Local directory: /tmp/dask-worker-space/worker-ps4fpt8x

Comm: tcp://127.0.0.1:33070	Total threads: 4
Dashboard: http://127.0.0.1:43675/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:37026
Local directory: /tmp/dask-worker-space/worker-yr9v6nhi

Comm: tcp://127.0.0.1:38283	Total threads: 4
Dashboard: http://127.0.0.1:43331/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:43664
Local directory: /tmp/dask-worker-space/worker-eggt3x5h

Comm: tcp://127.0.0.1:42958	Total threads: 4
Dashboard: http://127.0.0.1:45158/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:35139
Local directory: /tmp/dask-worker-space/worker-qoy0vuvp

Comm: tcp://127.0.0.1:44821	Total threads: 4
Dashboard: http://127.0.0.1:38799/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40103
Local directory: /tmp/dask-worker-space/worker-yvq1k3ds

If you submit the job with job scheduler, above¶

%env local='2"¶

%env ychunk='2'¶

%env tchunk='2'¶

controls chunk. 'False' sets no modification from original netcdf file's chunk.¶

ychunk=10 will group the original netcdf file to 10 by 10¶

tchunk=1 will chunk the time coordinate one by one¶

%env file_exp=¶

'file_exp': Which 'experiment' name is it?¶

. this corresopnds to intake catalog name without path and .yaml¶

%env year=¶

for Validation, this correspoinds to path/year/month 's year¶

for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory¶

setting it as 0[0-9] &1[0-9]& *[2-3][0-9], the job can be separated in three lots.¶

%env month=¶

for monitoring this corresponds to file path path-XIOS.{month}/¶

%env control=FWC_SSH¶

name of control file to be used for computation/plots/save/ & how it is called from Monitor.sh¶

%env save= proceed saving? True or False , Default is setted as True¶

%env plot= proceed plotting? True or False , Default is setted as True¶

%env calc= proceed computation? or just load computed result? True or False , Default is setted as True¶

%env save=False¶

%env lazy=False¶

For debugging this cell can help¶

0[1-2]¶

False¶

M_Sectiony ok with ychunk=False local=True lazy=False¶

Client

Cluster Info

LocalCluster

Scheduler Info

Scheduler

Workers

Worker: 0

Worker: 1

Worker: 2

Worker: 3

Worker: 4

Worker: 5

Worker: 6

Worker: 7

Worker: 8

Worker: 9

Worker: 10

Worker: 11

Worker: 12

Worker: 13

Worker: 14

Worker: 15

Worker: 16

Worker: 17

Worker: 18

Worker: 19

Worker: 20

Worker: 21

Worker: 22

Worker: 23

Worker: 24

Worker: 25

Worker: 26

Worker: 27

Worker: 28

Worker: 29

Worker: 30

Worker: 31

Worker: 32

Worker: 33

Worker: 34

Worker: 35

Worker: 36

Worker: 37

Worker: 38

Worker: 39

Worker: 40

Worker: 41

Worker: 42

Worker: 43

Worker: 44

Worker: 45

Worker: 46

Comm: tcp://127.0.0.1:33715	Total threads: 4
Dashboard: http://127.0.0.1:39851/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:34490
Local directory: /tmp/dask-worker-space/worker-74os2tr_

Comm: tcp://127.0.0.1:35421	Total threads: 4
Dashboard: http://127.0.0.1:34398/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:42493
Local directory: /tmp/dask-worker-space/worker-dfil_3v7

Comm: tcp://127.0.0.1:43020	Total threads: 4
Dashboard: http://127.0.0.1:44069/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:38287
Local directory: /tmp/dask-worker-space/worker-btxhcy68

Comm: tcp://127.0.0.1:45559	Total threads: 4
Dashboard: http://127.0.0.1:40147/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:41890
Local directory: /tmp/dask-worker-space/worker-tm91nkry

Comm: tcp://127.0.0.1:45348	Total threads: 4
Dashboard: http://127.0.0.1:34332/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:41638
Local directory: /tmp/dask-worker-space/worker-oh3z4q8f

Comm: tcp://127.0.0.1:32890	Total threads: 4
Dashboard: http://127.0.0.1:33139/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:35015
Local directory: /tmp/dask-worker-space/worker-x2edokum

Comm: tcp://127.0.0.1:43591	Total threads: 4
Dashboard: http://127.0.0.1:39800/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:44626
Local directory: /tmp/dask-worker-space/worker-kee5d3id

Comm: tcp://127.0.0.1:39998	Total threads: 4
Dashboard: http://127.0.0.1:39725/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:45207
Local directory: /tmp/dask-worker-space/worker-8atr6jf4

Comm: tcp://127.0.0.1:42887	Total threads: 4
Dashboard: http://127.0.0.1:34615/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:32972
Local directory: /tmp/dask-worker-space/worker-6j_x3ye1

Comm: tcp://127.0.0.1:45822	Total threads: 4
Dashboard: http://127.0.0.1:34572/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:46410
Local directory: /tmp/dask-worker-space/worker-eib9fwfp