%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()

from core import  load, zoom, calc, save,plots,monitor

#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)

<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>

# 'month':  = 'JOBID' almost month but not really, 

# If you submit the job with job scheduler, above

#below are list of enviroment variable one can pass
#%env local='2"
# local : if True  run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.  
# ychunk=10 will group the original netcdf file to 10 by 10 
# tchunk=1 will chunk the time coordinate one by one
#%env control=FWC_SSH 
# name of control file to be used for computation/plots/save/ 
#%env file_exp= 
# 'file_exp': Which 'experiment' name is it? 
#.    this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date'  having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring  this corresponds to file path path-XIOS.{month}/
#
#%env save=   proceed saving?   True or False  , Default is setted as True 
#%env plot=   proceed plotting?  True or False , Default is setted as True 
#%env calc=   proceed computation? or just load computed result? True or False , Default is setted as True 
#%env save=False

%%time
# 'savefig': Do we save output in html? or not. keep it true. 
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client

local True
using host= irene4520.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16
10000000000
False
not local in tgcc
rome local cluster starting
This code is running on  irene4520.c-irene.mg1.tgcc.ccc.cea.fr using  SEDNA_DELTA_MONITOR file experiment, read from  ../lib/SEDNA_DELTA_MONITOR.yaml  on year= 2012  on month= 02  outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6253905irene4520.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_02M_IceThick/
CPU times: user 348 ms, sys: 228 ms, total: 575 ms
Wall time: 11.5 s

Client Scheduler: tcp://127.0.0.1:45973 Dashboard: http://127.0.0.1:8787/status

Cluster Workers: 16 Cores: 256 Memory: 269.57 GB

read plotting information from a csv file

df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df

	Value	Inputs	Equation	Zone	Plot	Colourmap	MinMax	Unit	Oldname	Unnamed: 10
	IceThickness	icemod.sivolu	(data.sivolu.where(data.sivolu >0)).to_dataset...	ALL	maps	Spectral	(0,5)	m	M-4

Computation starts here

Each computation consists of

1. Load NEMO data set
2. Zoom data set
3. Compute (or load computed data set)
4. Save
5. Plot
6. Close

%%time
import os
calcswitch=os.environ.get('calc', 'True') 
loaddata=((df.Inputs != '').any()) 
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport) if ((calcswitch=='True' )*loaddata) else 0 
data

calcswitch= True df.Inputs != nothing True
../lib/SEDNA_DELTA_MONITOR.yaml
using param_xios reading  ../lib/SEDNA_DELTA_MONITOR.yaml
using param_xios reading  <bound method DataSourceBase.describe of sources:
  param_xios:
    args:
      combine: by_coords
      concat_dim: y
      urlpath: /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc
      xarray_kwargs:
        compat: override
        coords: minimal
        data_vars: minimal
        parallel: true
    description: SEDNA NEMO parameters from MPI output  nav_lon lat fails
    driver: intake_xarray.netcdf.NetCDFSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
{'name': 'param_xios', 'container': 'xarray', 'plugin': ['netcdf'], 'driver': ['netcdf'], 'description': 'SEDNA NEMO parameters from MPI output  nav_lon lat fails', 'direct_access': 'forbid', 'user_parameters': [{'name': 'path', 'description': 'file coordinate', 'type': 'str', 'default': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1/param'}], 'metadata': {}, 'args': {'urlpath': '/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32/x_*.nc', 'combine': 'by_coords', 'concat_dim': 'y'}}
0 read icemod ['sivolu']
using load_data_xios reading  icemod
using load_data_xios reading  <bound method DataSourceBase.describe of sources:
  data_xios:
    args:
      combine: by_coords
      concat_dim: time_counter,x,y
      urlpath: /ccc/scratch/cont003/gen7420/talandel/SEDNA/SEDNA-DELTA-S/SPLIT/1d/2012/02/*icemod_*_0[0-5][0-9][0-9].nc
      xarray_kwargs:
        compat: override
        coords: minimal
        data_vars: minimal
        drop_variables: !!set
          botpres: null
          deptht_bounds: null
          depthu_bounds: null
          iicestru: null
          iicestrv: null
          intstrx: null
          intstry: null
          mldkz5: null
          rhop_sig0: null
          siages: null
          sidive: null
          sisali: null
          sishea: null
          sistre: null
          sitemp: null
          snthic: null
          snvolu: null
          sometauy: null
          sozotaux: null
          time_centered_bounds: null
          time_counter_bounds: null
          utau_atmoce: null
          utau_iceoce: null
          uwspd10: null
          vtau_atmoce: null
          vtau_iceoce: null
          vwspd10: null
        parallel: true
        preprocess: !!python/name:core.load.prep ''
    description: SEDNA NEMO outputs from different xios server
    driver: intake_xarray.netcdf.NetCDFSource
    metadata:
      catalog_dir: /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/../lib/
>
      took 70.26886582374573 seconds
0 merging icemod ['sivolu']
param mask2d will be included in data
param nav_lat will be included in data
param nav_lon will be included in data
sum_num (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12)
start rechunking with (130, 122, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 48)
end of y_rechunk
CPU times: user 13.8 s, sys: 2.35 s, total: 16.2 s
Wall time: 1min 28s

<xarray.Dataset>
Dimensions:  (t: 28, x: 6560, y: 6540)
Coordinates:
    nav_lat  (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
    nav_lon  (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
  * t        (t) object 2012-02-01 12:00:00 ... 2012-02-28 12:00:00
  * y        (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540
  * x        (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560
    mask2d   (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray>
Data variables:
    sivolu   (t, y, x) float32 dask.array<chunksize=(28, 130, 6560), meta=np.ndarray>
Attributes:
    name:                    /ccc/scratch/cont003/ra5563/talandel/ONGOING-RUN...
    description:             ice variables
    title:                   ice variables
    Conventions:             CF-1.6
    timeStamp:               2022-Jan-18 16:51:17 GMT
    uuid:                    56b165e2-bdda-4b33-a2e9-04a59f3d06e9
    ibegin:                  0
    ni:                      6560
    jbegin:                  0
    nj:                      13
    DOMAIN_number_total:     544
    DOMAIN_number:           0
    DOMAIN_dimensions_ids:   [2 3]
    DOMAIN_size_global:      [6560 6540]
    DOMAIN_size_local:       [6560   13]
    DOMAIN_position_first:   [1 1]
    DOMAIN_position_last:    [6560   13]
    DOMAIN_halo_size_start:  [0 0]
    DOMAIN_halo_size_end:    [0 0]
    DOMAIN_type:             box
    start_date:              20090101
    output_frequency:        1d
    CONFIG:                  SEDNA
    CASE:                    DELTA
    history:                 Wed Jan 19 12:40:39 2022: ncks -4 -L 1 SEDNA-DEL...
    NCO:                     netCDF Operators version 4.9.1 (Homepage = http:...

%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
            )

#calc= True
#save= True
#plot= False
Zone='ALL'
Value='IceThickness'
cmap='Spectral'
clabel='m'
clim= (0, 5)
outputpath='../results/SEDNA_DELTA_MONITOR/'
nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/'
filename='SEDNA_maps_ALL_IceThickness'
#3 Start computing 
dtaa= (data.sivolu.where(data.sivolu >0)).to_dataset(name='sivolu').chunk({ 't': -1 }).unify_chunks().persist()

<xarray.Dataset>
Dimensions:  (t: 28, x: 6560, y: 6540)
Coordinates:
    nav_lat  (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
    nav_lon  (y, x) float32 dask.array<chunksize=(130, 6560), meta=np.ndarray>
  * t        (t) object 2012-02-01 12:00:00 ... 2012-02-28 12:00:00
  * y        (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540
  * x        (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560
    mask2d   (y, x) bool dask.array<chunksize=(130, 6560), meta=np.ndarray>
Data variables:
    sivolu   (t, y, x) float32 dask.array<chunksize=(28, 130, 6560), meta=np.ndarray>

#4 Saving  SEDNA_maps_ALL_IceThickness
dtaa=save.datas(data,plot=Plot,path=nc_outputpath,filename=filename)
start saving data
saving data in a file
t (28,)
0
slice(0, 28, None)
CPU times: user 12.2 s, sys: 8 s, total: 20.2 s
Wall time: 31 s