%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
below are list of enviroment variable one can pass
local : if True run dask local cluster, if not true, put number of workers setted in the 'local' if no 'local ' given, local will be setted automatically to 'True'
#
Monitor.sh calls M_MLD_2D
and AWTD.sh, Fluxnet.sh, Siconc.sh, IceClim.sh, FWC_SSH.sh
AWTD.sh M_AWTMD
Fluxnet.sh M_Fluxnet
FWC_SSH.sh M_FWC_2D M_FWC_integrals M_FWC_SSH M_SSH_anomaly
Integrals.sh M_Mean_temp_velo M_Mooring M_Sectionx M_Sectiony
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local True using host= irene4353.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False tgcc local cluster starting This code is running on irene4353.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_DELTA_MONITOR file experiment, read from ../lib/SEDNA_DELTA_MONITOR.yaml on year= 2012 on month= 01 outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6419279irene4353.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_01M_Mooring/ CPU times: user 3.89 s, sys: 805 ms, total: 4.7 s Wall time: 1min 40s
Client-bf6e5194-13da-11ed-8424-080038b93395
Connection method: Cluster object | Cluster type: distributed.LocalCluster |
Dashboard: http://127.0.0.1:8787/status |
066b8121
Dashboard: http://127.0.0.1:8787/status | Workers: 64 |
Total threads: 256 | Total memory: 251.06 GiB |
Status: running | Using processes: True |
Scheduler-a4a74b22-fa14-4e46-b5c8-d5f76b835756
Comm: tcp://127.0.0.1:45666 | Workers: 64 |
Dashboard: http://127.0.0.1:8787/status | Total threads: 256 |
Started: 1 minute ago | Total memory: 251.06 GiB |
Comm: tcp://127.0.0.1:36499 | Total threads: 4 |
Dashboard: http://127.0.0.1:42875/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:40339 | |
Local directory: /tmp/dask-worker-space/worker-giloff1l |
Comm: tcp://127.0.0.1:44268 | Total threads: 4 |
Dashboard: http://127.0.0.1:40836/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37946 | |
Local directory: /tmp/dask-worker-space/worker-0g5j3l5y |
Comm: tcp://127.0.0.1:33307 | Total threads: 4 |
Dashboard: http://127.0.0.1:34512/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37983 | |
Local directory: /tmp/dask-worker-space/worker-l5omtpie |
Comm: tcp://127.0.0.1:45228 | Total threads: 4 |
Dashboard: http://127.0.0.1:45861/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:46106 | |
Local directory: /tmp/dask-worker-space/worker-lek5tdl1 |
Comm: tcp://127.0.0.1:33269 | Total threads: 4 |
Dashboard: http://127.0.0.1:35033/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44158 | |
Local directory: /tmp/dask-worker-space/worker-y92gttzd |
Comm: tcp://127.0.0.1:43174 | Total threads: 4 |
Dashboard: http://127.0.0.1:44211/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:46360 | |
Local directory: /tmp/dask-worker-space/worker-5sneg_78 |
Comm: tcp://127.0.0.1:35486 | Total threads: 4 |
Dashboard: http://127.0.0.1:33851/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44767 | |
Local directory: /tmp/dask-worker-space/worker-xjh8xxrl |
Comm: tcp://127.0.0.1:41111 | Total threads: 4 |
Dashboard: http://127.0.0.1:40235/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:39201 | |
Local directory: /tmp/dask-worker-space/worker-_i2zppjb |
Comm: tcp://127.0.0.1:34667 | Total threads: 4 |
Dashboard: http://127.0.0.1:43408/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37030 | |
Local directory: /tmp/dask-worker-space/worker-nqc1wsw9 |
Comm: tcp://127.0.0.1:46424 | Total threads: 4 |
Dashboard: http://127.0.0.1:42479/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:43981 | |
Local directory: /tmp/dask-worker-space/worker-4jlaebz2 |
Comm: tcp://127.0.0.1:34727 | Total threads: 4 |
Dashboard: http://127.0.0.1:42093/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:38032 | |
Local directory: /tmp/dask-worker-space/worker-4npktg6t |
Comm: tcp://127.0.0.1:33943 | Total threads: 4 |
Dashboard: http://127.0.0.1:45873/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:36586 | |
Local directory: /tmp/dask-worker-space/worker-tyaeekyr |
Comm: tcp://127.0.0.1:41368 | Total threads: 4 |
Dashboard: http://127.0.0.1:34410/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44252 | |
Local directory: /tmp/dask-worker-space/worker-tltf1j9b |
Comm: tcp://127.0.0.1:42896 | Total threads: 4 |
Dashboard: http://127.0.0.1:44924/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:38342 | |
Local directory: /tmp/dask-worker-space/worker-5mwirf61 |
Comm: tcp://127.0.0.1:33413 | Total threads: 4 |
Dashboard: http://127.0.0.1:34999/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:36205 | |
Local directory: /tmp/dask-worker-space/worker-5ek6wi4z |
Comm: tcp://127.0.0.1:40985 | Total threads: 4 |
Dashboard: http://127.0.0.1:46409/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:39367 | |
Local directory: /tmp/dask-worker-space/worker-y6ozq7pk |
Comm: tcp://127.0.0.1:41018 | Total threads: 4 |
Dashboard: http://127.0.0.1:45629/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:35870 | |
Local directory: /tmp/dask-worker-space/worker-oooldfqi |
Comm: tcp://127.0.0.1:45978 | Total threads: 4 |
Dashboard: http://127.0.0.1:35456/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:36907 | |
Local directory: /tmp/dask-worker-space/worker-ipakr4zv |
Comm: tcp://127.0.0.1:37031 | Total threads: 4 |
Dashboard: http://127.0.0.1:42396/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:35310 | |
Local directory: /tmp/dask-worker-space/worker-hscaqd4w |
Comm: tcp://127.0.0.1:38047 | Total threads: 4 |
Dashboard: http://127.0.0.1:43792/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:33934 | |
Local directory: /tmp/dask-worker-space/worker-n8h2h49l |
Comm: tcp://127.0.0.1:41342 | Total threads: 4 |
Dashboard: http://127.0.0.1:43269/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:39616 | |
Local directory: /tmp/dask-worker-space/worker-979q0gpa |
Comm: tcp://127.0.0.1:41364 | Total threads: 4 |
Dashboard: http://127.0.0.1:46641/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:33914 | |
Local directory: /tmp/dask-worker-space/worker-k1ocb7lc |
Comm: tcp://127.0.0.1:41166 | Total threads: 4 |
Dashboard: http://127.0.0.1:43183/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:42369 | |
Local directory: /tmp/dask-worker-space/worker-sv0z3p95 |
Comm: tcp://127.0.0.1:45389 | Total threads: 4 |
Dashboard: http://127.0.0.1:44105/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:42813 | |
Local directory: /tmp/dask-worker-space/worker-g1nw7xif |
Comm: tcp://127.0.0.1:42355 | Total threads: 4 |
Dashboard: http://127.0.0.1:35541/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44659 | |
Local directory: /tmp/dask-worker-space/worker-pfj7y1yh |
Comm: tcp://127.0.0.1:37389 | Total threads: 4 |
Dashboard: http://127.0.0.1:41058/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:45080 | |
Local directory: /tmp/dask-worker-space/worker-b_28y_u9 |
Comm: tcp://127.0.0.1:37002 | Total threads: 4 |
Dashboard: http://127.0.0.1:43697/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44891 | |
Local directory: /tmp/dask-worker-space/worker-olnj4um5 |
Comm: tcp://127.0.0.1:33418 | Total threads: 4 |
Dashboard: http://127.0.0.1:37952/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:39518 | |
Local directory: /tmp/dask-worker-space/worker-6ilbc7_s |
Comm: tcp://127.0.0.1:44088 | Total threads: 4 |
Dashboard: http://127.0.0.1:44538/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:40592 | |
Local directory: /tmp/dask-worker-space/worker-lrw_1_27 |
Comm: tcp://127.0.0.1:43147 | Total threads: 4 |
Dashboard: http://127.0.0.1:43807/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:34539 | |
Local directory: /tmp/dask-worker-space/worker-jqur7_3q |
Comm: tcp://127.0.0.1:33896 | Total threads: 4 |
Dashboard: http://127.0.0.1:42284/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:39814 | |
Local directory: /tmp/dask-worker-space/worker-dzl_zczf |
Comm: tcp://127.0.0.1:38353 | Total threads: 4 |
Dashboard: http://127.0.0.1:36342/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44729 | |
Local directory: /tmp/dask-worker-space/worker-9iuvzp6j |
Comm: tcp://127.0.0.1:35697 | Total threads: 4 |
Dashboard: http://127.0.0.1:35067/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44700 | |
Local directory: /tmp/dask-worker-space/worker-rnau2t0d |
Comm: tcp://127.0.0.1:40603 | Total threads: 4 |
Dashboard: http://127.0.0.1:45694/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:39365 | |
Local directory: /tmp/dask-worker-space/worker-uia9zusq |
Comm: tcp://127.0.0.1:44814 | Total threads: 4 |
Dashboard: http://127.0.0.1:35143/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:34886 | |
Local directory: /tmp/dask-worker-space/worker-ptzrm7q3 |
Comm: tcp://127.0.0.1:44284 | Total threads: 4 |
Dashboard: http://127.0.0.1:33267/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:40768 | |
Local directory: /tmp/dask-worker-space/worker-j_9cm1rz |
Comm: tcp://127.0.0.1:37802 | Total threads: 4 |
Dashboard: http://127.0.0.1:43086/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44665 | |
Local directory: /tmp/dask-worker-space/worker-mwi1dv7p |
Comm: tcp://127.0.0.1:33994 | Total threads: 4 |
Dashboard: http://127.0.0.1:33612/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37935 | |
Local directory: /tmp/dask-worker-space/worker-f44c_ojm |
Comm: tcp://127.0.0.1:37176 | Total threads: 4 |
Dashboard: http://127.0.0.1:42274/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:45003 | |
Local directory: /tmp/dask-worker-space/worker-xo2ku9fv |
Comm: tcp://127.0.0.1:38813 | Total threads: 4 |
Dashboard: http://127.0.0.1:39198/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37466 | |
Local directory: /tmp/dask-worker-space/worker-utd7fkx5 |
Comm: tcp://127.0.0.1:38308 | Total threads: 4 |
Dashboard: http://127.0.0.1:36602/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:40165 | |
Local directory: /tmp/dask-worker-space/worker-lmb62wrb |
Comm: tcp://127.0.0.1:36082 | Total threads: 4 |
Dashboard: http://127.0.0.1:32910/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:34405 | |
Local directory: /tmp/dask-worker-space/worker-hc22vfs1 |
Comm: tcp://127.0.0.1:38113 | Total threads: 4 |
Dashboard: http://127.0.0.1:42662/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:41059 | |
Local directory: /tmp/dask-worker-space/worker-y4tggqm1 |
Comm: tcp://127.0.0.1:46109 | Total threads: 4 |
Dashboard: http://127.0.0.1:36456/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:34196 | |
Local directory: /tmp/dask-worker-space/worker-nku7wur8 |
Comm: tcp://127.0.0.1:42481 | Total threads: 4 |
Dashboard: http://127.0.0.1:44398/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:35352 | |
Local directory: /tmp/dask-worker-space/worker-38od8p_6 |
Comm: tcp://127.0.0.1:46877 | Total threads: 4 |
Dashboard: http://127.0.0.1:34945/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:36436 | |
Local directory: /tmp/dask-worker-space/worker-jrkac_5g |
Comm: tcp://127.0.0.1:40319 | Total threads: 4 |
Dashboard: http://127.0.0.1:45310/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44024 | |
Local directory: /tmp/dask-worker-space/worker-qcrj5hsq |
Comm: tcp://127.0.0.1:34532 | Total threads: 4 |
Dashboard: http://127.0.0.1:45489/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:41711 | |
Local directory: /tmp/dask-worker-space/worker-z56pqimy |
Comm: tcp://127.0.0.1:33891 | Total threads: 4 |
Dashboard: http://127.0.0.1:40081/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:41464 | |
Local directory: /tmp/dask-worker-space/worker-_9z6leb3 |
Comm: tcp://127.0.0.1:34326 | Total threads: 4 |
Dashboard: http://127.0.0.1:44951/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44045 | |
Local directory: /tmp/dask-worker-space/worker-h0ifk0r4 |
Comm: tcp://127.0.0.1:41694 | Total threads: 4 |
Dashboard: http://127.0.0.1:43723/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:38096 | |
Local directory: /tmp/dask-worker-space/worker-wndjf9_h |
Comm: tcp://127.0.0.1:36821 | Total threads: 4 |
Dashboard: http://127.0.0.1:38066/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:43127 | |
Local directory: /tmp/dask-worker-space/worker-9vhbn3gu |
Comm: tcp://127.0.0.1:36893 | Total threads: 4 |
Dashboard: http://127.0.0.1:46784/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37580 | |
Local directory: /tmp/dask-worker-space/worker-1uk8qby6 |
Comm: tcp://127.0.0.1:45989 | Total threads: 4 |
Dashboard: http://127.0.0.1:37559/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:41481 | |
Local directory: /tmp/dask-worker-space/worker-g7jl1dyn |
Comm: tcp://127.0.0.1:32866 | Total threads: 4 |
Dashboard: http://127.0.0.1:41712/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44793 | |
Local directory: /tmp/dask-worker-space/worker-j7edcszp |
Comm: tcp://127.0.0.1:42483 | Total threads: 4 |
Dashboard: http://127.0.0.1:39988/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:45114 | |
Local directory: /tmp/dask-worker-space/worker-9q7iuoh7 |
Comm: tcp://127.0.0.1:44895 | Total threads: 4 |
Dashboard: http://127.0.0.1:45745/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:44091 | |
Local directory: /tmp/dask-worker-space/worker-cg23cc9d |
Comm: tcp://127.0.0.1:37300 | Total threads: 4 |
Dashboard: http://127.0.0.1:35536/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:43883 | |
Local directory: /tmp/dask-worker-space/worker-5p4_wqxn |
Comm: tcp://127.0.0.1:41898 | Total threads: 4 |
Dashboard: http://127.0.0.1:43323/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:36881 | |
Local directory: /tmp/dask-worker-space/worker-8zy8hele |
Comm: tcp://127.0.0.1:39276 | Total threads: 4 |
Dashboard: http://127.0.0.1:37505/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:34335 | |
Local directory: /tmp/dask-worker-space/worker-vikd7bkx |
Comm: tcp://127.0.0.1:46827 | Total threads: 4 |
Dashboard: http://127.0.0.1:43531/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:37556 | |
Local directory: /tmp/dask-worker-space/worker-mnx2_0xw |
Comm: tcp://127.0.0.1:41672 | Total threads: 4 |
Dashboard: http://127.0.0.1:40214/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:38767 | |
Local directory: /tmp/dask-worker-space/worker-s3qginwp |
Comm: tcp://127.0.0.1:46325 | Total threads: 4 |
Dashboard: http://127.0.0.1:36204/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:45762 | |
Local directory: /tmp/dask-worker-space/worker-uq8wal2f |
Comm: tcp://127.0.0.1:42086 | Total threads: 4 |
Dashboard: http://127.0.0.1:42814/status | Memory: 3.92 GiB |
Nanny: tcp://127.0.0.1:34393 | |
Local directory: /tmp/dask-worker-space/worker-_1tg6zfs |
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
Mooring_Arc_B | gridS.vosaline,gridT.votemper,param.depth,para... | data | Arc_B | Mooring | rainbow | {'vosaline': (28.0,34.4), 'votemper': (-2.0,2.0)} | None | x | ||
Mooring_Eur_B | gridS.vosaline,gridT.votemper,param.depth,para... | data | Eur_B | Mooring | rainbow | {'vosaline': (32.0,35.0), 'votemper': (-2.0,3.0)} | None | x |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
lazy=os.environ.get('lazy','False' )
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True lazy= False ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading ../lib/SEDNA_DELTA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: nested concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}} 0 read gridS ['vosaline'] lazy= False using load_data_xios_kerchunk reading gridS using load_data_xios_kerchunk reading <bound method DataSourceBase.describe of sources: data_xios_kerchunk: args: consolidated: false storage_options: fo: file:////ccc/cont003/home/ra5563/ra5563/catalogue/DELTA/201201/gridS_0[0-5][0-9][0-9].json target_protocol: file urlpath: reference:// description: CREG025 NEMO outputs from different xios server in kerchunk format driver: intake_xarray.xzarr.ZarrSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 23.693634748458862 seconds 0 merging gridS ['vosaline'] 1 read gridT ['votemper'] lazy= False using load_data_xios_kerchunk reading gridT using load_data_xios_kerchunk reading <bound method DataSourceBase.describe of sources: data_xios_kerchunk: args: consolidated: false storage_options: fo: file:////ccc/cont003/home/ra5563/ra5563/catalogue/DELTA/201201/gridT_0[0-5][0-9][0-9].json target_protocol: file urlpath: reference:// description: CREG025 NEMO outputs from different xios server in kerchunk format driver: intake_xarray.xzarr.ZarrSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 22.180362224578857 seconds 1 merging gridT ['votemper'] took 0.9276597499847412 seconds param mask2d will be included in data param depth will be included in data param nav_lon will be included in data param nav_lat will be included in data param mask will be included in data ychunk= 5 calldatas_y_rechunk sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12) start rechunking with (65, 65, 62, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 48) end of y_rechunk before rechunking t item (1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) start rechunking t with 1 end of t_rechunk CPU times: user 49.7 s, sys: 13.3 s, total: 1min 2s Wall time: 1min 44s
<xarray.Dataset> Dimensions: (t: 31, z: 150, y: 6540, x: 6560) Coordinates: time_centered (t) object dask.array<chunksize=(1,), meta=np.ndarray> * t (t) object 2012-01-01 12:00:00 ... 2012-01-31 12:00:00 * y (y) int64 1 2 3 4 5 6 7 ... 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 ... 6555 6556 6557 6558 6559 6560 * z (z) int64 1 2 3 4 5 6 7 8 ... 143 144 145 146 147 148 149 150 mask2d (y, x) bool dask.array<chunksize=(65, 6560), meta=np.ndarray> depth (z, y, x) float32 dask.array<chunksize=(150, 65, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(65, 6560), meta=np.ndarray> nav_lat (y, x) float32 dask.array<chunksize=(65, 6560), meta=np.ndarray> mask (z, y, x) bool dask.array<chunksize=(150, 65, 6560), meta=np.ndarray> Data variables: vosaline (t, z, y, x) float32 dask.array<chunksize=(1, 150, 65, 6560), meta=np.ndarray> votemper (t, z, y, x) float32 dask.array<chunksize=(1, 150, 65, 6560), meta=np.ndarray> Attributes: (12/26) CASE: DELTA CONFIG: SEDNA Conventions: CF-1.6 DOMAIN_dimensions_ids: [2, 3] DOMAIN_halo_size_end: [0, 0] DOMAIN_halo_size_start: [0, 0] ... ... nj: 13 output_frequency: 1d start_date: 20090101 timeStamp: 2022-Jan-17 19:00:16 GMT title: ocean T grid variables uuid: d8db76f6-a436-451a-9ab1-72dc892753af
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
#calc= True #save= True #plot= False Value='Mooring_Arc_B' Zone='Arc_B' Plot='Mooring' cmap='rainbow' clabel='None' clim= {'vosaline': (28.0, 34.4), 'votemper': (-2.0, 2.0)} outputpath='../results/SEDNA_DELTA_MONITOR/' nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/' filename='SEDNA_Mooring_Arc_B_Mooring_Arc_B' data=monitor.optimize_dataset(data) #2 Zooming Data data= zoom.Arc_B(data) data=monitor.optimize_dataset(data)
<xarray.Dataset> Dimensions: (t: 31, z: 102) Coordinates: time_centered (t) object dask.array<chunksize=(1,), meta=np.ndarray> * t (t) object 2012-01-01 12:00:00 ... 2012-01-31 12:00:00 y int64 4967 x int64 2471 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 94 95 96 97 98 99 100 101 102 mask2d bool dask.array<chunksize=(), meta=np.ndarray> depth (z) float32 dask.array<chunksize=(102,), meta=np.ndarray> nav_lon float32 dask.array<chunksize=(), meta=np.ndarray> nav_lat float32 dask.array<chunksize=(), meta=np.ndarray> mask (z) bool dask.array<chunksize=(102,), meta=np.ndarray> Data variables: vosaline (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> votemper (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> Attributes: (12/26) CASE: DELTA CONFIG: SEDNA Conventions: CF-1.6 DOMAIN_dimensions_ids: [2, 3] DOMAIN_halo_size_end: [0, 0] DOMAIN_halo_size_start: [0, 0] ... ... nj: 13 output_frequency: 1d start_date: 20090101 timeStamp: 2022-Jan-17 19:00:16 GMT title: ocean T grid variables uuid: d8db76f6-a436-451a-9ab1-72dc892753af
#3 Start computing data= data monitor.optimize_dataset(data) add optimise here once otimise can recognise
<xarray.Dataset> Dimensions: (t: 31, z: 102) Coordinates: time_centered (t) object dask.array<chunksize=(1,), meta=np.ndarray> * t (t) object 2012-01-01 12:00:00 ... 2012-01-31 12:00:00 y int64 4967 x int64 2471 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 94 95 96 97 98 99 100 101 102 mask2d bool dask.array<chunksize=(), meta=np.ndarray> depth (z) float32 dask.array<chunksize=(102,), meta=np.ndarray> nav_lon float32 dask.array<chunksize=(), meta=np.ndarray> nav_lat float32 dask.array<chunksize=(), meta=np.ndarray> mask (z) bool dask.array<chunksize=(102,), meta=np.ndarray> Data variables: vosaline (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> votemper (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> Attributes: (12/26) CASE: DELTA CONFIG: SEDNA Conventions: CF-1.6 DOMAIN_dimensions_ids: [2, 3] DOMAIN_halo_size_end: [0, 0] DOMAIN_halo_size_start: [0, 0] ... ... nj: 13 output_frequency: 1d start_date: 20090101 timeStamp: 2022-Jan-17 19:00:16 GMT title: ocean T grid variables uuid: d8db76f6-a436-451a-9ab1-72dc892753af
#4 Saving SEDNA_Mooring_Arc_B_Mooring_Arc_B data=save.datas(data,plot=Plot,path=nc_outputpath,filename=filename) start saving data saving data in a csv file ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_Mooring_Arc_B_Mooring_Arc_B2012-01-01_2012-01-31.nc save computed data at ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_Mooring_Arc_B_Mooring_Arc_B2012-01-01_2012-01-31.nc completed Value='Mooring_Eur_B' Zone='Eur_B' Plot='Mooring' cmap='rainbow' clabel='None' clim= {'vosaline': (32.0, 35.0), 'votemper': (-2.0, 3.0)} outputpath='../results/SEDNA_DELTA_MONITOR/' nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/' filename='SEDNA_Mooring_Eur_B_Mooring_Eur_B' data=monitor.optimize_dataset(data) #2 Zooming Data data= zoom.Eur_B(data) data=monitor.optimize_dataset(data)
<xarray.Dataset> Dimensions: (t: 31, z: 102) Coordinates: time_centered (t) object dask.array<chunksize=(1,), meta=np.ndarray> * t (t) object 2012-01-01 12:00:00 ... 2012-01-31 12:00:00 y int64 3621 x int64 2495 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 94 95 96 97 98 99 100 101 102 mask2d bool dask.array<chunksize=(), meta=np.ndarray> depth (z) float32 dask.array<chunksize=(102,), meta=np.ndarray> nav_lon float32 dask.array<chunksize=(), meta=np.ndarray> nav_lat float32 dask.array<chunksize=(), meta=np.ndarray> mask (z) bool dask.array<chunksize=(102,), meta=np.ndarray> Data variables: vosaline (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> votemper (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> Attributes: (12/26) CASE: DELTA CONFIG: SEDNA Conventions: CF-1.6 DOMAIN_dimensions_ids: [2, 3] DOMAIN_halo_size_end: [0, 0] DOMAIN_halo_size_start: [0, 0] ... ... nj: 13 output_frequency: 1d start_date: 20090101 timeStamp: 2022-Jan-17 19:00:16 GMT title: ocean T grid variables uuid: d8db76f6-a436-451a-9ab1-72dc892753af
#3 Start computing data= data monitor.optimize_dataset(data) add optimise here once otimise can recognise
<xarray.Dataset> Dimensions: (t: 31, z: 102) Coordinates: time_centered (t) object dask.array<chunksize=(1,), meta=np.ndarray> * t (t) object 2012-01-01 12:00:00 ... 2012-01-31 12:00:00 y int64 3621 x int64 2495 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 94 95 96 97 98 99 100 101 102 mask2d bool dask.array<chunksize=(), meta=np.ndarray> depth (z) float32 dask.array<chunksize=(102,), meta=np.ndarray> nav_lon float32 dask.array<chunksize=(), meta=np.ndarray> nav_lat float32 dask.array<chunksize=(), meta=np.ndarray> mask (z) bool dask.array<chunksize=(102,), meta=np.ndarray> Data variables: vosaline (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> votemper (t, z) float32 dask.array<chunksize=(1, 102), meta=np.ndarray> Attributes: (12/26) CASE: DELTA CONFIG: SEDNA Conventions: CF-1.6 DOMAIN_dimensions_ids: [2, 3] DOMAIN_halo_size_end: [0, 0] DOMAIN_halo_size_start: [0, 0] ... ... nj: 13 output_frequency: 1d start_date: 20090101 timeStamp: 2022-Jan-17 19:00:16 GMT title: ocean T grid variables uuid: d8db76f6-a436-451a-9ab1-72dc892753af
#4 Saving SEDNA_Mooring_Eur_B_Mooring_Eur_B data=save.datas(data,plot=Plot,path=nc_outputpath,filename=filename) start saving data saving data in a csv file ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_Mooring_Eur_B_Mooring_Eur_B2012-01-01_2012-01-31.nc save computed data at ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_Mooring_Eur_B_Mooring_Eur_B2012-01-01_2012-01-31.nc completed CPU times: user 13.8 s, sys: 2.52 s, total: 16.3 s Wall time: 26.8 s