%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
# 'month': = 'JOBID' almost month but not really,
# If you submit the job with job scheduler, above
#below are list of enviroment variable one can pass
#%env local='2"
# local : if True run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.
# ychunk=10 will group the original netcdf file to 10 by 10
# tchunk=1 will chunk the time coordinate one by one
#%env control=FWC_SSH
# name of control file to be used for computation/plots/save/
#%env file_exp=
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date' having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring this corresponds to file path path-XIOS.{month}/
#
#%env save= proceed saving? True or False , Default is setted as True
#%env plot= proceed plotting? True or False , Default is setted as True
#%env calc= proceed computation? or just load computed result? True or False , Default is setted as True
#%env save=False
#%env lazy=False
%%time
# 'savefig': Do we save output in html? or not. keep it true.
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client
local 20 local false ,workers 20 using host= irene5032.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= False workers 20 10000000000 False mg1: rome dask jobqueue rome, slurmcluster starting This code is running on irene5032.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_DELTA_MONITOR file experiment, read from ../lib/SEDNA_DELTA_MONITOR.yaml on year= 2012 on month= 0[1-2] outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6413866irene5032.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_0[1-2]M_MLD_2D/_lazyTrue CPU times: user 16.8 s, sys: 4.19 s, total: 21 s Wall time: 21min 12s
Client-c70b0c8a-1346-11ed-892e-080038b93d7d
| Connection method: Cluster object | Cluster type: dask_jobqueue.SLURMCluster |
| Dashboard: http://10.131.6.171:8787/status |
18d470aa
| Dashboard: http://10.131.6.171:8787/status | Workers: 28 |
| Total threads: 252 | Total memory: 428.40 GiB |
Scheduler-40124d7f-73be-4d1f-92e5-2e097ecf3ec5
| Comm: tcp://10.131.6.171:34156 | Workers: 28 |
| Dashboard: http://10.131.6.171:8787/status | Total threads: 252 |
| Started: 21 minutes ago | Total memory: 428.40 GiB |
| Comm: tcp://10.131.5.48:38486 | Total threads: 9 |
| Dashboard: http://10.131.5.48:41134/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:39731 | |
| Local directory: /tmp/dask-worker-space/worker-kxl45o8v | |
| Comm: tcp://10.131.5.48:43620 | Total threads: 9 |
| Dashboard: http://10.131.5.48:45923/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:35737 | |
| Local directory: /tmp/dask-worker-space/worker-al64v5jh | |
| Comm: tcp://10.131.5.48:43317 | Total threads: 9 |
| Dashboard: http://10.131.5.48:37400/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:45842 | |
| Local directory: /tmp/dask-worker-space/worker-rfj86s0h | |
| Comm: tcp://10.131.5.48:36633 | Total threads: 9 |
| Dashboard: http://10.131.5.48:34299/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:44669 | |
| Local directory: /tmp/dask-worker-space/worker-p3qbcqwa | |
| Comm: tcp://10.131.5.48:33890 | Total threads: 9 |
| Dashboard: http://10.131.5.48:34219/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:44533 | |
| Local directory: /tmp/dask-worker-space/worker-wqkf_hid | |
| Comm: tcp://10.131.5.48:38577 | Total threads: 9 |
| Dashboard: http://10.131.5.48:34820/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:45140 | |
| Local directory: /tmp/dask-worker-space/worker-xlptivvs | |
| Comm: tcp://10.131.5.48:36145 | Total threads: 9 |
| Dashboard: http://10.131.5.48:37153/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:42298 | |
| Local directory: /tmp/dask-worker-space/worker-wgocpjci | |
| Comm: tcp://10.131.5.48:45268 | Total threads: 9 |
| Dashboard: http://10.131.5.48:35782/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:35763 | |
| Local directory: /tmp/dask-worker-space/worker-1j_79nqg | |
| Comm: tcp://10.131.5.48:42374 | Total threads: 9 |
| Dashboard: http://10.131.5.48:39971/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:40477 | |
| Local directory: /tmp/dask-worker-space/worker-1b1ei4zq | |
| Comm: tcp://10.131.5.48:36846 | Total threads: 9 |
| Dashboard: http://10.131.5.48:39516/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:40838 | |
| Local directory: /tmp/dask-worker-space/worker-gzmxp06t | |
| Comm: tcp://10.131.5.48:36490 | Total threads: 9 |
| Dashboard: http://10.131.5.48:33202/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:43991 | |
| Local directory: /tmp/dask-worker-space/worker-cffp247u | |
| Comm: tcp://10.131.5.48:36653 | Total threads: 9 |
| Dashboard: http://10.131.5.48:37381/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:38525 | |
| Local directory: /tmp/dask-worker-space/worker-80foeenk | |
| Comm: tcp://10.131.5.48:36379 | Total threads: 9 |
| Dashboard: http://10.131.5.48:37940/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:38431 | |
| Local directory: /tmp/dask-worker-space/worker-3851qv0z | |
| Comm: tcp://10.131.5.48:43521 | Total threads: 9 |
| Dashboard: http://10.131.5.48:39017/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.5.48:34045 | |
| Local directory: /tmp/dask-worker-space/worker-urpp5fwr | |
| Comm: tcp://10.131.9.175:46075 | Total threads: 9 |
| Dashboard: http://10.131.9.175:35688/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:34517 | |
| Local directory: /tmp/dask-worker-space/worker-czjm65u5 | |
| Comm: tcp://10.131.9.175:45807 | Total threads: 9 |
| Dashboard: http://10.131.9.175:35625/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:45805 | |
| Local directory: /tmp/dask-worker-space/worker-ryje5dkr | |
| Comm: tcp://10.131.9.175:43092 | Total threads: 9 |
| Dashboard: http://10.131.9.175:38029/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:33676 | |
| Local directory: /tmp/dask-worker-space/worker-5uo8q_im | |
| Comm: tcp://10.131.9.175:34062 | Total threads: 9 |
| Dashboard: http://10.131.9.175:35449/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:34069 | |
| Local directory: /tmp/dask-worker-space/worker-wwjemn94 | |
| Comm: tcp://10.131.9.175:34890 | Total threads: 9 |
| Dashboard: http://10.131.9.175:42956/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:33647 | |
| Local directory: /tmp/dask-worker-space/worker-itn338yt | |
| Comm: tcp://10.131.9.175:45241 | Total threads: 9 |
| Dashboard: http://10.131.9.175:39143/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:36170 | |
| Local directory: /tmp/dask-worker-space/worker-069j05xq | |
| Comm: tcp://10.131.9.175:38957 | Total threads: 9 |
| Dashboard: http://10.131.9.175:40678/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:34744 | |
| Local directory: /tmp/dask-worker-space/worker-ru8okux7 | |
| Comm: tcp://10.131.9.175:35222 | Total threads: 9 |
| Dashboard: http://10.131.9.175:44426/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:35996 | |
| Local directory: /tmp/dask-worker-space/worker-48mtcnon | |
| Comm: tcp://10.131.9.175:38975 | Total threads: 9 |
| Dashboard: http://10.131.9.175:36392/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:45755 | |
| Local directory: /tmp/dask-worker-space/worker-2f9fbzw_ | |
| Comm: tcp://10.131.9.175:37048 | Total threads: 9 |
| Dashboard: http://10.131.9.175:34895/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:37365 | |
| Local directory: /tmp/dask-worker-space/worker-oqoozgm_ | |
| Comm: tcp://10.131.9.175:42856 | Total threads: 9 |
| Dashboard: http://10.131.9.175:34330/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:42090 | |
| Local directory: /tmp/dask-worker-space/worker-oc7nv5yy | |
| Comm: tcp://10.131.9.175:36033 | Total threads: 9 |
| Dashboard: http://10.131.9.175:33298/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:39306 | |
| Local directory: /tmp/dask-worker-space/worker-4vfnegzo | |
| Comm: tcp://10.131.9.175:46574 | Total threads: 9 |
| Dashboard: http://10.131.9.175:34233/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:35245 | |
| Local directory: /tmp/dask-worker-space/worker-0fthzw9m | |
| Comm: tcp://10.131.9.175:43932 | Total threads: 9 |
| Dashboard: http://10.131.9.175:39956/status | Memory: 15.30 GiB |
| Nanny: tcp://10.131.9.175:45335 | |
| Local directory: /tmp/dask-worker-space/worker-5j64hkyz | |
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
| Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
|---|---|---|---|---|---|---|---|---|---|---|
| MLD_2D | gridT-2D.samldr1_1 | data.samldr1_1 | ALL | maps | Blues | (0,80) | m | M-5 |
Each computation consists of
%%time
import os
calcswitch=os.environ.get('calc', 'True')
lazy=os.environ.get('lazy','True' )
loaddata=((df.Inputs != '').any())
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0
data
calcswitch= True df.Inputs != nothing True lazy= True
../lib/SEDNA_DELTA_MONITOR.yaml
using param_xios reading ../lib/SEDNA_DELTA_MONITOR.yaml
using param_xios reading <bound method DataSourceBase.describe of sources:
param_xios:
args:
combine: nested
concat_dim: y
urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc
xarray_kwargs:
compat: override
coords: minimal
data_vars: minimal
parallel: true
description: SEDNA NEMO parameters from MPI output nav_lon lat fails
driver: intake_xarray.netcdf.NetCDFSource
metadata:
catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
{'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'nested', 'concat_dim': 'y'}}
0 read gridT-2D ['samldr1_1']
using load_data_xios reading gridT-2D
using load_data_xios reading {'name': 'data_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO outputs from different xios server', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'name of config', 'type': 'str', 'default': '/ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d'}, {'name': 'fileexp', 'description': 'name of config', 'type': 'str', 'default': 'SEDNA-DELTA'}, {'name': 'month', 'description': 'running number 2 digit', 'type': 'str', 'default': '02'}, {'name': 'freq', 'description': '1d or 1m', 'type': 'str', 'default': '1d'}, {'name': 'year', 'description': 'last digits of yearmonthdate.', 'type': 'str', 'default': '2012'}, {'name': 'file', 'description': 'file name', 'type': 'str', 'default': 'icemod'}, {'name': 'eio', 'description': 'xios mpi number', 'type': 'str', 'default': '0[0-5][0-9][0-9]'}], 'metadata': {}, 'args': {'urlpath': '{{path}}/{{year}}/{{month}}/*{{file}}_*_{{eio}}.nc', 'combine': 'nested', 'concat_dim': 'time_counter,x,y'}}
took 173.4489085674286 seconds
0 merging gridT-2D ['samldr1_1']
param nav_lat will be included in data
param mask2d will be included in data
param nav_lon will be included in data
ychunk= 10 calldatas_y_rechunk
sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12)
start rechunking with (130, 122, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 48)
end of y_rechunk
CPU times: user 59.6 s, sys: 7.68 s, total: 1min 7s
Wall time: 3min 15s
<xarray.Dataset>
Dimensions: (t: 59, y: 6540, x: 6560)
Coordinates:
* t (t) object 2012-01-01 12:00:00 ... 2012-02-28 12:00:00
* y (y) int64 1 2 3 4 5 6 7 ... 6535 6536 6537 6538 6539 6540
* x (x) int64 1 2 3 4 5 6 7 ... 6555 6556 6557 6558 6559 6560
nav_lat (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
nav_lon (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
time_centered (t) object dask.array<chunksize=(59,), meta=np.ndarray>
mask2d (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray>
Data variables:
samldr1_1 (t, y, x) float32 dask.array<chunksize=(59, 130, 6560), meta=np.ndarray>
Attributes: (12/26)
name: /ccc/scratch/cont003/ra5563/talandel/ONGOING-RUN...
description: ocean T grid variables
title: ocean T grid variables
Conventions: CF-1.6
timeStamp: 2022-Jan-17 19:00:11 GMT
uuid: 88aad392-7ae9-43ca-8ed2-c5606b75d4a0
... ...
start_date: 20090101
output_frequency: 1d
CONFIG: SEDNA
CASE: DELTA
history: Tue Jan 18 19:00:08 2022: ncks -4 -L 1 SEDNA-DEL...
NCO: netCDF Operators version 4.9.1 (Homepage = http:...%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
)
#calc= True #save= False #plot= True monitor.optimize_dataset(data) Value='MLD_2D' Zone='ALL' Plot='maps' cmap='Blues' clabel='m' clim= (0, 80) outputpath='../results/SEDNA_DELTA_MONITOR/' nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/' filename='SEDNA_maps_ALL_MLD_2D' #3 Start computing dtaa= data.samldr1_1
--------------------------------------------------------------------------- AttributeError Traceback (most recent call last) File <timed eval>:1, in <module> File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py:75, in auto(df, val, savefig, daskreport, outputpath, file_exp) 73 print('#3 no computing , loading starts') 74 print('dtaa=save.load_data(plot=Plot,path=nc_outputpath,filename=filename)' ) ---> 75 with performance_report(filename=daskreport+"_calc_"+step.Value+".html"): 76 data=save.load_data(plot=Plot,path=nc_outputpath,filename=filename) 77 #saveswitch=False File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py:100, in optimize_dataset(ds) 97 display(IFrame(filename, width=1000, height=500)) 98 return --> 100 def optimize_dataset(ds): 101 import dask 102 for varname, da in ds.data_vars.items(): 103 #print(varname) File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/core/common.py:256, in AttrAccessMixin.__getattr__(self, name) 254 with suppress(KeyError): 255 return source[name] --> 256 raise AttributeError( 257 f"{type(self).__name__!r} object has no attribute {name!r}" 258 ) AttributeError: 'DataArray' object has no attribute 'data_vars'