
%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()

from core import  load, zoom, calc, save,plots,monitor


#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)

<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>


%%time
# 'savefig': Do we save output in html? or not. keep it true. 
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client

local True
using host= irene6193.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16
10000000000
False
tgcc local cluster starting
This code is running on  irene6193.c-irene.mg1.tgcc.ccc.cea.fr using  SEDNA_DELTA_MONITOR file experiment, read from  ../lib/SEDNA_DELTA_MONITOR.yaml  on year= 2012  on month= 02  outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6419109irene6193.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_02M_IceClim/
CPU times: user 3.83 s, sys: 693 ms, total: 4.52 s
Wall time: 1min 32s


df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df


%%time
import os
calcswitch=os.environ.get('calc', 'True') 
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any()) 
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0 
data

calcswitch= True df.Inputs != nothing False lazy= True
CPU times: user 399 µs, sys: 0 ns, total: 399 µs
Wall time: 399 µs

0


%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
            )

#calc= True
#save= False
#plot= True
Value='IceClim'
Zone='ALL'
Plot='IceClim'
cmap='Spectral'
clabel='m'
clim= (0, 5)
outputpath='../results/SEDNA_DELTA_MONITOR/'
nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/'
filename='SEDNA_IceClim_ALL_IceClim'
#3 Start computing 
data= calc.IceClim_load(data,nc_outputpath)
monitor.optimize_dataset(data)
start saving data
filename= ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_maps_ALL_IceConce/t_*/y_*/x_*.nc

---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
File <timed eval>:1, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py:67, in auto(df, val, savefig, daskreport, outputpath, file_exp)
     65 #print('count:',data.count())
     66 with performance_report(filename=daskreport+"_calc_"+step.Value+".html"):
---> 67     data=eval(command)
     68     #print('persist ')
     69     #data=data.persist()
     70     print('add optimise here once otimise can recognise')

File <string>:1, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/calc.py:231, in IceClim_load(data, nc_outputpath)
    229 import xarray as xr
    230 filename='SEDNA_maps_ALL_IceConce'
--> 231 ds=save.load_data(plot='map',path=nc_outputpath,filename=filename)
    232 filename='SEDNA_maps_ALL_IceThickness'
    233 ds['sivolu']=save.load_data(plot='map',path=nc_outputpath,filename=filename).sivolu

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py:38, in load_data(plot, path, filename)
     36     data=load_twoD(path,filename,nested=False)    
     37 else:
---> 38     data=load_twoD(path,filename)        
     39 print('load computed data completed')    
     40 return data

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py:48, in load_twoD(path, filename, nested)
     46 dim=('x','y','t') if nested else ('t')
     47 print ('filename=',filename)
---> 48 return xr.open_mfdataset(filename,parallel=True
     49               ,compat='override'
     50               ,data_vars='minimal'
     51               ,concat_dim=dim
     52               ,combine='nested' #param_xios
     53               ,coords='minimal')

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:937, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
    934     paths = [os.fspath(p) if isinstance(p, os.PathLike) else p for p in paths]
    936 if not paths:
--> 937     raise OSError("no files to open")
    939 if combine == "nested":
    940     if isinstance(concat_dim, (str, DataArray)) or concat_dim is None:

OSError: no files to open

Dashboard: http://127.0.0.1:8787/status	Workers: 64
Total threads: 256	Total memory: 251.06 GiB
Status: running	Using processes: True

Comm: tcp://127.0.0.1:37458	Workers: 64
Dashboard: http://127.0.0.1:8787/status	Total threads: 256
Started: 1 minute ago	Total memory: 251.06 GiB

Comm: tcp://127.0.0.1:45957	Total threads: 4
Dashboard: http://127.0.0.1:39026/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:34194
Local directory: /tmp/dask-worker-space/worker-uagzzxbi

Comm: tcp://127.0.0.1:34934	Total threads: 4
Dashboard: http://127.0.0.1:44715/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:33835
Local directory: /tmp/dask-worker-space/worker-ywge_oxo

Comm: tcp://127.0.0.1:40389	Total threads: 4
Dashboard: http://127.0.0.1:33307/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:37971
Local directory: /tmp/dask-worker-space/worker-62824nfm

Connection method: Cluster object	Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Comm: tcp://127.0.0.1:33667	Total threads: 4
Dashboard: http://127.0.0.1:46603/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40247
Local directory: /tmp/dask-worker-space/worker-c07ul4o2

Comm: tcp://127.0.0.1:44033	Total threads: 4
Dashboard: http://127.0.0.1:35614/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:39715
Local directory: /tmp/dask-worker-space/worker-1mp0p7fe

Comm: tcp://127.0.0.1:38697	Total threads: 4
Dashboard: http://127.0.0.1:44252/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:46182
Local directory: /tmp/dask-worker-space/worker-0oqfjll8

Comm: tcp://127.0.0.1:38209	Total threads: 4
Dashboard: http://127.0.0.1:40379/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40685
Local directory: /tmp/dask-worker-space/worker-umkmxwjm

Comm: tcp://127.0.0.1:40113	Total threads: 4
Dashboard: http://127.0.0.1:45385/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:32945
Local directory: /tmp/dask-worker-space/worker-v5csb2u8

Comm: tcp://127.0.0.1:34031	Total threads: 4
Dashboard: http://127.0.0.1:43852/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:45340
Local directory: /tmp/dask-worker-space/worker-yfskcops

Comm: tcp://127.0.0.1:43843	Total threads: 4
Dashboard: http://127.0.0.1:45178/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:42856
Local directory: /tmp/dask-worker-space/worker-a_49chq1

Comm: tcp://127.0.0.1:42857	Total threads: 4
Dashboard: http://127.0.0.1:41087/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:34264
Local directory: /tmp/dask-worker-space/worker-afajyut1

Comm: tcp://127.0.0.1:33579	Total threads: 4
Dashboard: http://127.0.0.1:39458/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:43224
Local directory: /tmp/dask-worker-space/worker-jthxykki

Comm: tcp://127.0.0.1:33729	Total threads: 4
Dashboard: http://127.0.0.1:40684/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:44284
Local directory: /tmp/dask-worker-space/worker-x58rv_du

Comm: tcp://127.0.0.1:37389	Total threads: 4
Dashboard: http://127.0.0.1:44493/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:36729
Local directory: /tmp/dask-worker-space/worker-sequ2xeu

Comm: tcp://127.0.0.1:45033	Total threads: 4
Dashboard: http://127.0.0.1:36305/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:36577
Local directory: /tmp/dask-worker-space/worker-p_9me8vi

Comm: tcp://127.0.0.1:41214	Total threads: 4
Dashboard: http://127.0.0.1:43231/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:36120
Local directory: /tmp/dask-worker-space/worker-iomfxe19

Comm: tcp://127.0.0.1:43997	Total threads: 4
Dashboard: http://127.0.0.1:43498/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:42181
Local directory: /tmp/dask-worker-space/worker-_og7lrm9

Comm: tcp://127.0.0.1:35935	Total threads: 4
Dashboard: http://127.0.0.1:39053/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:41986
Local directory: /tmp/dask-worker-space/worker-pixjzkcg

Comm: tcp://127.0.0.1:33949	Total threads: 4
Dashboard: http://127.0.0.1:39203/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:40973
Local directory: /tmp/dask-worker-space/worker-yne41vdk

Comm: tcp://127.0.0.1:32855	Total threads: 4
Dashboard: http://127.0.0.1:44575/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:44735
Local directory: /tmp/dask-worker-space/worker-eablhvp_

Comm: tcp://127.0.0.1:43051	Total threads: 4
Dashboard: http://127.0.0.1:34097/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:39701
Local directory: /tmp/dask-worker-space/worker-85pyr01_

Comm: tcp://127.0.0.1:39298	Total threads: 4
Dashboard: http://127.0.0.1:46537/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:34487
Local directory: /tmp/dask-worker-space/worker-yned81v6

If you submit the job with job scheduler, above¶

%env local='2"¶

%env ychunk='2'¶

%env tchunk='2'¶

controls chunk. 'False' sets no modification from original netcdf file's chunk.¶

ychunk=10 will group the original netcdf file to 10 by 10¶

tchunk=1 will chunk the time coordinate one by one¶

%env file_exp=¶

'file_exp': Which 'experiment' name is it?¶

. this corresopnds to intake catalog name without path and .yaml¶

%env year=¶

for Validation, this correspoinds to path/year/month 's year¶

for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory¶

setting it as 0[0-9] &1[0-9]& *[2-3][0-9], the job can be separated in three lots.¶

%env month=¶

for monitoring this corresponds to file path path-XIOS.{month}/¶

%env control=FWC_SSH¶

name of control file to be used for computation/plots/save/ & how it is called from Monitor.sh¶

%env save= proceed saving? True or False , Default is setted as True¶

%env plot= proceed plotting? True or False , Default is setted as True¶

%env calc= proceed computation? or just load computed result? True or False , Default is setted as True¶

%env save=False¶

%env lazy=False¶

For debugging this cell can help¶

0[1-2]¶

False¶

M_Sectiony ok with ychunk=False local=True lazy=False¶

Client

Cluster Info

LocalCluster

Scheduler Info

Scheduler

Workers

Worker: 0

Worker: 1

Worker: 2

Worker: 3

Worker: 4

Worker: 5

Worker: 6

Worker: 7

Worker: 8

Worker: 9

Worker: 10

Worker: 11

Worker: 12

Worker: 13

Worker: 14

Worker: 15

Worker: 16

Worker: 17

Worker: 18

Worker: 19

Worker: 20

Worker: 21

Worker: 22

Worker: 23

Worker: 24

Worker: 25

Worker: 26

Worker: 27

Worker: 28

Worker: 29

Worker: 30

Worker: 31

Worker: 32

Worker: 33

Worker: 34

Worker: 35

Worker: 36

Worker: 37

Worker: 38

Worker: 39

Worker: 40

Worker: 41

Worker: 42

Worker: 43

Worker: 44

Worker: 45

Worker: 46

Comm: tcp://127.0.0.1:34722	Total threads: 4
Dashboard: http://127.0.0.1:46821/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:46324
Local directory: /tmp/dask-worker-space/worker-e4xatcck

Comm: tcp://127.0.0.1:46553	Total threads: 4
Dashboard: http://127.0.0.1:34718/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:43540
Local directory: /tmp/dask-worker-space/worker-ot7ton3x

Comm: tcp://127.0.0.1:43430	Total threads: 4
Dashboard: http://127.0.0.1:34142/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:39644
Local directory: /tmp/dask-worker-space/worker-oxol206d

Comm: tcp://127.0.0.1:44821	Total threads: 4
Dashboard: http://127.0.0.1:43227/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:45086
Local directory: /tmp/dask-worker-space/worker-m9l3a3le

Comm: tcp://127.0.0.1:35801	Total threads: 4
Dashboard: http://127.0.0.1:34597/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:37190
Local directory: /tmp/dask-worker-space/worker-1dh612tx

Comm: tcp://127.0.0.1:32956	Total threads: 4
Dashboard: http://127.0.0.1:33890/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:42475
Local directory: /tmp/dask-worker-space/worker-3imnbx2n

Comm: tcp://127.0.0.1:34658	Total threads: 4
Dashboard: http://127.0.0.1:36878/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:34785
Local directory: /tmp/dask-worker-space/worker-xu_4tdw8

Comm: tcp://127.0.0.1:35568	Total threads: 4
Dashboard: http://127.0.0.1:42898/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:32889
Local directory: /tmp/dask-worker-space/worker-y7s_yhhp

Comm: tcp://127.0.0.1:41721	Total threads: 4
Dashboard: http://127.0.0.1:42631/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:45672
Local directory: /tmp/dask-worker-space/worker-asiuuvjk

Comm: tcp://127.0.0.1:38207	Total threads: 4
Dashboard: http://127.0.0.1:32994/status	Memory: 3.92 GiB
Nanny: tcp://127.0.0.1:41406
Local directory: /tmp/dask-worker-space/worker-iwvaqglg