%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()

from core import  load, zoom, calc, save,plots,monitor

#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)

<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>

# 'catalog_url': we chose intake catalog for reading.
# 'month':  = 'JOBID' almost month but not really, 
# 'savefig': Do we save output in html? or not.
# 'file_exp': Which 'experiment' name is it? 
#.    this corresopnds to catalog name without path and .yaml
# If you submit the job with job scheduler, above
# 4 valus can be passed as enviroment variable.  
# local : if True  run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
# %env local='2"
#%env ychunk='2'


#%env tchunk='2'
#%env local='True'
#local=os.environ.get('local',"True" )

#if ('True' in local): print(local) 

%%time

savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client

local True
using host= irene5500.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16
10000000000
False
not local in tgcc
rome local cluster starting
This code is running on  irene5500.c-irene.mg1.tgcc.ccc.cea.fr using  SEDNA_ALPHA_MONITOR file experiment, read from  ../lib/SEDNA_ALPHA_MONITOR.yaml  on year= *  on month= 22  outputpath= ../results/rome_SEDNA_ALPHA_MONITOR/22/ daskreport= ../results/dask/2459381irene5500.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_ALPHA_MONITOR_22compute/
CPU times: user 340 ms, sys: 239 ms, total: 579 ms
Wall time: 9.28 s

Client Scheduler: tcp://127.0.0.1:34180 Dashboard: http://127.0.0.1:8787/status

Cluster Workers: 16 Cores: 256 Memory: 269.57 GB

read plotting information from a csv file

df=load.controlfile(control)
#
#df=df[~df.duplicated(subset='Plot' )]
df=df[df['Plot'] == 'AWTD_map']
#df=df[3:4]
df

	Value	Inputs	Equation	Zone	Plot	Colourmap	MinMax	Unit	Oldname	Unnamed: 10
	AW_maxtemp_depth	gridT.votemper,gridS.vosaline,param.mask,param...	calc.AWTD4(data)	ALL	AWTD_map	jet	(0,800)	m	M-5

Computation starts here

Each computation consists of

1. Create data set
2. Zoom Data set
3. Compute
4. (Save)
5. Plot
6. Close

%%time
#todo add 'year' here.
data=load.datas(catalog_url,df.Inputs,month,year,daskreport) 

print('#1 Data: created:')
print('# if we raed too much file, we can do sel to take out some datess..')
data

../lib/SEDNA_ALPHA_MONITOR.yaml
using param_xios reading  ../lib/SEDNA_ALPHA_MONITOR.yaml
using param_xios reading  <bound method DataSourceBase.describe of sources:
  param_xios:
    args:
      combine: by_coords
      concat_dim: y
      urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc
      xarray_kwargs:
        compat: override
        coords: minimal
        data_vars: minimal
        parallel: true
    description: SEDNA NEMO parameters from MPI output  nav_lon lat fails
    driver: intake_xarray.netcdf.NetCDFSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
{'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output  nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}}
0 read gridS ['vosaline']
using load_data_xios reading  gridS
using load_data_xios reading  <bound method DataSourceBase.describe of sources:
  data_xios:
    args:
      combine: by_coords
      concat_dim: time_counter,x,y
      urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridS_*_0[0-5][0-9][0-9].nc
      xarray_kwargs:
        compat: override
        coords: minimal
        data_vars: minimal
        drop_variables: !!set
          deptht_bounds: null
          depthu_bounds: null
          nav_lat: null
          nav_lon: null
          time_centerd: null
          time_centered_bounds: null
          time_counter_bounds: null
        parallel: true
        preprocess: !!python/name:core.load.prep ''
    description: SEDNA NEMO outputs from different xios server
    driver: intake_xarray.netcdf.NetCDFSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
      took 152.70950865745544 seconds
0 merging gridS ['vosaline']
1 read gridT ['votemper']
using load_data_xios reading  gridT
using load_data_xios reading  <bound method DataSourceBase.describe of sources:
  data_xios:
    args:
      combine: by_coords
      concat_dim: time_counter,x,y
      urlpath: /ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22/SEDNA-ALPHA_1d_gridT_*_0[0-5][0-9][0-9].nc
      xarray_kwargs:
        compat: override
        coords: minimal
        data_vars: minimal
        drop_variables: !!set
          deptht_bounds: null
          depthu_bounds: null
          nav_lat: null
          nav_lon: null
          time_centerd: null
          time_centered_bounds: null
          time_counter_bounds: null
        parallel: true
        preprocess: !!python/name:core.load.prep ''
    description: SEDNA NEMO outputs from different xios server
    driver: intake_xarray.netcdf.NetCDFSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
      took 75.66219639778137 seconds
1 merging gridT ['votemper']
      took 0.33994603157043457 seconds
param depth will be included in data
param mask2d will be included in data
param mask will be included in data
param nav_lat will be included in data
param nav_lon will be included in data
sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12)
start rechunking with (130, 122, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 48)
end of y_rechunk
#1 Data: created:
# if we raed too much file, we can do sel to take out some datess..
CPU times: user 1min 40s, sys: 17.5 s, total: 1min 57s
Wall time: 4min

<xarray.Dataset>
Dimensions:   (t: 15, x: 6560, y: 6540, z: 150)
Coordinates:
  * t         (t) object 2004-06-16 12:00:00 ... 2004-06-30 12:00:00
  * y         (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540
  * x         (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560
  * z         (z) int64 1 2 3 4 5 6 7 8 9 ... 143 144 145 146 147 148 149 150
    depth     (z, y, x) float64 dask.array<chunksize=(150, 130, 6560), meta=np.ndarray>
    mask2d    (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray>
    mask      (z, y, x) bool dask.array<chunksize=(150, 130, 6560), meta=np.ndarray>
    nav_lat   (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
    nav_lon   (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
Data variables:
    vosaline  (t, z, y, x) float32 dask.array<chunksize=(1, 150, 130, 6560), meta=np.ndarray>
    votemper  (t, z, y, x) float32 dask.array<chunksize=(1, 150, 130, 6560), meta=np.ndarray>

%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
            )

switch:calcswitch,saveswitch,plotswitch True False True
dtaa= calc.AWTD4(data) #3 Start computing 
count: <xarray.Dataset>
Dimensions:   ()
Data variables:
    vosaline  int64 dask.array<chunksize=(), meta=np.ndarray>
    votemper  int64 dask.array<chunksize=(), meta=np.ndarray>

<xarray.Dataset>
Dimensions:   ()
Data variables:
    vosaline  int64 dask.array<chunksize=(), meta=np.ndarray>
    votemper  int64 dask.array<chunksize=(), meta=np.ndarray>

nbytes: 8108658520
count: <xarray.Dataset>
Dimensions:  ()
Data variables:
    AWT      int64 dask.array<chunksize=(), meta=np.ndarray>
    AWD      int64 dask.array<chunksize=(), meta=np.ndarray>

<xarray.Dataset>
Dimensions:  (t: 15, x: 6560, y: 6540)
Coordinates:
  * t        (t) object 2004-06-16 12:00:00 ... 2004-06-30 12:00:00
  * y        (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540
  * x        (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560
    mask2d   (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray>
    nav_lat  (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
    nav_lon  (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
Data variables:
    AWT      (t, y, x) float32 dask.array<chunksize=(1, 130, 6560), meta=np.ndarray>
    AWD      (t, y, x) float64 dask.array<chunksize=(1, 130, 6560), meta=np.ndarray>

plots.AWTD_map(data,path=outputpath,filename=filename,save=savefig,cmap='jet',clim=(0,800),clabel='m') #5 Plotting SEDNA_AWTD_map_ALL_AW_maxtemp_depth
../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_AWTD_map_ALL_AW_maxtemp_depth_20040616-20040630.html starts plotting
../results/rome_SEDNA_ALPHA_MONITOR/22/SEDNA_AWTD_map_ALL_AW_maxtemp_depth_20040616-20040630.html created

CPU times: user 1h 41min 39s, sys: 4min 46s, total: 1h 46min 26s
Wall time: 1h 58min 14s