%matplotlib inline
#import xarray as xr
import pandas as pd
import socket
host = socket.getfqdn()
#import sys
#sys.path.append(os.path.abspath("..")
from core import load, zoom, calc, save,plots,monitor
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
client, cluster=load.daskcluster(host,local=True)
#uncomment and use below only if you want to use dask
# worker on the same node as you run jupyter notebook
#
#client, cluster=load.daskcluster(host,local=True)
client
using host= irene5202.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True 10000000000 False not local in tgcc rome local cluster starting
Client
|
Cluster
|
# 'catalog_url': we chose intake catalog for reading.
# 'month': = 'JOBID' almost month but not really,
# 'savefig': Do we save output in html? or not.
# 'file_exp': Which 'experiment' name is it?
#. this corresopnds to catalog name without path and .yaml
# If you submit the job with job scheduler, above
# 4 valus can be passed as enviroment variable.
#
import os
os.environ['file_exp']='SEDNA_ALPHA_MONITOR'
os.environ['month']="22"
os.environ['year']="*[3-4]" #"*0"
%%time
savefig=True
control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
This code is running on irene5202.c-irene.mg1.tgcc.ccc.cea.fr using SEDNA_ALPHA_MONITOR file experiment, read from ../lib/SEDNA_ALPHA_MONITOR.yaml on year= *[3-4] on month= 22 outputpath= ../results/rome_SEDNA_ALPHA_MONITOR/22/ daskreport= ../results/dask/irene5202.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_22/compute CPU times: user 18.6 ms, sys: 13.5 ms, total: 32.2 ms Wall time: 341 ms
df=load.controlfile(control)
#
#df=df[~df.duplicated(subset='Plot' )]
df=df[df['Plot'] == 'maps']
df=df[3:4]
df
Value | Inputs | Equation | Zone | Plot | Colourmap | MinMax | Unit | Oldname | Unnamed: 10 | |
---|---|---|---|---|---|---|---|---|---|---|
AW_maxtemp_depth | gridT.votemper,gridS.vosaline,param.mask,param... | calc.AWD(data) | ALL | maps | jet | (0,800) | m | M-5 |
Each computation consists of
%%time
#todo add 'year' here.
data=load.datas(catalog_url,df.Inputs,month,year,daskreport)
print('#1 Data: created:')
print('# if we raed too much file, we can do sel to take out some datess..')
data
../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading ../lib/SEDNA_ALPHA_MONITOR.yaml using param_xios reading <bound method DataSourceBase.describe of sources: param_xios: args: combine: by_coords concat_dim: y urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc xarray_kwargs: compat: override coords: minimal data_vars: minimal parallel: true description: SEDNA NEMO parameters from MPI output nav_lon lat fails driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > {'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}} 0 read gridS ['vosaline'] using load_data_xios reading gridS using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridS_*[3-4]_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 108.44777059555054 seconds 0 merging gridS ['vosaline'] 1 read gridT ['votemper'] using load_data_xios reading gridT using load_data_xios reading <bound method DataSourceBase.describe of sources: data_xios: args: combine: by_coords concat_dim: time_counter,x,y urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridT_*[3-4]_0[0-5][0-9][0-9].nc xarray_kwargs: compat: override coords: minimal data_vars: minimal drop_variables: !!set deptht_bounds: null depthu_bounds: null nav_lat: null nav_lon: null time_centerd: null time_centered_bounds: null time_counter_bounds: null parallel: true preprocess: !!python/name:core.load.prep '' description: SEDNA NEMO outputs from different xios server driver: intake_xarray.netcdf.NetCDFSource metadata: catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/ > took 12.120903253555298 seconds 1 merging gridT ['votemper'] took 0.43334126472473145 seconds param mask will be included in data param nav_lon will be included in data param depth will be included in data param mask2d will be included in data param nav_lat will be included in data #1 Data: created: # if we raed too much file, we can do sel to take out some datess.. CPU times: user 22.6 s, sys: 9.29 s, total: 31.9 s Wall time: 2min 16s
<xarray.Dataset> Dimensions: (t: 2, x: 6560, y: 6540, z: 150) Coordinates: * t (t) object 2004-06-23 12:00:00 2004-06-24 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 * z (z) int64 1 2 3 4 5 6 7 8 9 ... 143 144 145 146 147 148 149 150 mask (z, y, x) bool dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> depth (z, y, x) float64 dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lat (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> Data variables: vosaline (t, z, y, x) float32 dask.array<chunksize=(1, 150, 13, 6560), meta=np.ndarray> votemper (t, z, y, x) float32 dask.array<chunksize=(1, 150, 13, 6560), meta=np.ndarray>
array([cftime.DatetimeNoLeap(2004, 6, 23, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 24, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150])
|
|
|
|
|
|
|
#ds=data#.isel(t=slice(0,1))
ds=load.datas_rechunk(data,daskreport=daskreport,ysize=40)
sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12) start rechunking with (492, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 288) end of rechunk
import os
os.environ['save']='False'
os.environ['calc']='True'
os.environ['plot']='True'
%%time
monitor.auto(df,ds,savefig,daskreport,outputpath,file_exp='SEDNA'
)
switch:calcswitch,saveswitch,plotswitch True False True dtaa= calc.AWD(data) #3 Start computing count: <xarray.Dataset> Dimensions: () Data variables: vosaline int64 dask.array<chunksize=(), meta=np.ndarray> votemper int64 dask.array<chunksize=(), meta=np.ndarray>
<xarray.Dataset> Dimensions: () Data variables: vosaline int64 dask.array<chunksize=(), meta=np.ndarray> votemper int64 dask.array<chunksize=(), meta=np.ndarray>
|
|
nbytes: 1072664816 count: <xarray.Dataset> Dimensions: () Data variables: AWD int64 dask.array<chunksize=(), meta=np.ndarray>
<xarray.Dataset> Dimensions: (t: 2, x: 6560, y: 6540) Coordinates: * t (t) object 2004-06-23 12:00:00 2004-06-24 12:00:00 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 nav_lon (y, x) float32 dask.array<chunksize=(492, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(492, 6560), meta=np.ndarray> nav_lat (y, x) float32 dask.array<chunksize=(492, 6560), meta=np.ndarray> Data variables: AWD (t, y, x) float64 dask.array<chunksize=(1, 492, 6560), meta=np.ndarray>
array([cftime.DatetimeNoLeap(2004, 6, 23, 12, 0, 0, 0), cftime.DatetimeNoLeap(2004, 6, 24, 12, 0, 0, 0)], dtype=object)
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
|
|
|
|
plots.maps(data,path=outputpath,filename=filename,save=savefig,cmap='jet',clim=(0,800),clabel='m') #5 Plotting SEDNA_maps_ALL_AW_maxtemp_depth
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_maps_ALL_AW_maxtemp_depth_20040623-20040624.html starts plotting
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
0%| | 0/2 [00:00<?, ?it/s]
distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker distributed.nanny - WARNING - Worker exceeded 99% memory budget. Restarting distributed.nanny - WARNING - Restarting worker
../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_maps_ALL_AW_maxtemp_depth_20040623-20040624.html created
CPU times: user 2min 7s, sys: 24.4 s, total: 2min 31s Wall time: 9min 27s