In [1]:
%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()

from core import  load, zoom, calc, save,plots,monitor
In [2]:
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
Out[2]:
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
In [3]:
# 'month':  = 'JOBID' almost month but not really, 

# If you submit the job with job scheduler, above

#below are list of enviroment variable one can pass
#%env local='2"
# local : if True  run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.  
# ychunk=10 will group the original netcdf file to 10 by 10 
# tchunk=1 will chunk the time coordinate one by one
#%env control=FWC_SSH 
# name of control file to be used for computation/plots/save/ 
#%env file_exp= 
# 'file_exp': Which 'experiment' name is it? 
#.    this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date'  having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring  this corresponds to file path path-XIOS.{month}/
#
#%env save=   proceed saving?   True or False  , Default is setted as True 
#%env plot=   proceed plotting?  True or False , Default is setted as True 
#%env calc=   proceed computation? or just load computed result? True or False , Default is setted as True 
#%env save=False
#%env lazy=False
In [4]:
%env file_exp=SEDNA_DELTA_MONITOR
%env year=2012
%env month=02
%env ychunk=10
%env save=False
%env plot=True
%env calc=True
%env lazy=
#False
%env control=M_IceClim
#M_Sectiony ok with ychunk=False local=True  lazy=False 

env: file_exp=SEDNA_DELTA_MONITOR
env: year=2012
env: month=02
env: ychunk=10
env: save=False
env: plot=True
env: calc=True
env: lazy=
env: control=M_IceClim
In [5]:
%%time
# 'savefig': Do we save output in html? or not. keep it true. 
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client

local True
using host= irene5628.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16
10000000000
False
rome local cluster starting
This code is running on  irene5628.c-irene.mg1.tgcc.ccc.cea.fr using  SEDNA_DELTA_MONITOR file experiment, read from  ../lib/SEDNA_DELTA_MONITOR.yaml  on year= 2012  on month= 02  outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6414203irene5628.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_02M_IceClim/_lazy
CPU times: user 547 ms, sys: 127 ms, total: 674 ms
Wall time: 19.1 s
Out[5]:

Client

Client-107a4ca8-134a-11ed-8222-080038b94735

Connection method: Cluster object Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Cluster Info

LocalCluster

9bc03432

Dashboard: http://127.0.0.1:8787/status Workers: 16
Total threads: 128 Total memory: 251.06 GiB
Status: running Using processes: True

Scheduler Info

Scheduler

Scheduler-c556d06a-ac31-463a-8d29-0a5af9591ab1

Comm: tcp://127.0.0.1:33909 Workers: 16
Dashboard: http://127.0.0.1:8787/status Total threads: 128
Started: Just now Total memory: 251.06 GiB

Workers

Worker: 0
Comm: tcp://127.0.0.1:38312 Total threads: 8
Dashboard: http://127.0.0.1:36914/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:35585
Local directory: /tmp/dask-worker-space/worker-_moa8uc4

Worker: 1
Comm: tcp://127.0.0.1:38547 Total threads: 8
Dashboard: http://127.0.0.1:34308/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:42904
Local directory: /tmp/dask-worker-space/worker-wlqru5z6

Worker: 2
Comm: tcp://127.0.0.1:33939 Total threads: 8
Dashboard: http://127.0.0.1:35811/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:36628
Local directory: /tmp/dask-worker-space/worker-iylg8chc

Worker: 3
Comm: tcp://127.0.0.1:37487 Total threads: 8
Dashboard: http://127.0.0.1:40787/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:39102
Local directory: /tmp/dask-worker-space/worker-y688gnqn

Worker: 4
Comm: tcp://127.0.0.1:44853 Total threads: 8
Dashboard: http://127.0.0.1:43927/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:43025
Local directory: /tmp/dask-worker-space/worker-6biu8ry_

Worker: 5
Comm: tcp://127.0.0.1:38032 Total threads: 8
Dashboard: http://127.0.0.1:33484/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:33625
Local directory: /tmp/dask-worker-space/worker-d9_kwh49

Worker: 6
Comm: tcp://127.0.0.1:41725 Total threads: 8
Dashboard: http://127.0.0.1:44654/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:45379
Local directory: /tmp/dask-worker-space/worker-qa8szsvf

Worker: 7
Comm: tcp://127.0.0.1:40316 Total threads: 8
Dashboard: http://127.0.0.1:33134/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:43023
Local directory: /tmp/dask-worker-space/worker-u1tlo2pb

Worker: 8
Comm: tcp://127.0.0.1:36722 Total threads: 8
Dashboard: http://127.0.0.1:35900/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:33341
Local directory: /tmp/dask-worker-space/worker-akolis3d

Worker: 9
Comm: tcp://127.0.0.1:39198 Total threads: 8
Dashboard: http://127.0.0.1:34369/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:42254
Local directory: /tmp/dask-worker-space/worker-9k_26w9n

Worker: 10
Comm: tcp://127.0.0.1:37179 Total threads: 8
Dashboard: http://127.0.0.1:38310/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:41086
Local directory: /tmp/dask-worker-space/worker-g2fmah5n

Worker: 11
Comm: tcp://127.0.0.1:46722 Total threads: 8
Dashboard: http://127.0.0.1:35140/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:43104
Local directory: /tmp/dask-worker-space/worker-e9ywimuj

Worker: 12
Comm: tcp://127.0.0.1:42910 Total threads: 8
Dashboard: http://127.0.0.1:44970/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:36821
Local directory: /tmp/dask-worker-space/worker-teqx2mn7

Worker: 13
Comm: tcp://127.0.0.1:40713 Total threads: 8
Dashboard: http://127.0.0.1:34777/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:34116
Local directory: /tmp/dask-worker-space/worker-w6ay0hke

Worker: 14
Comm: tcp://127.0.0.1:36644 Total threads: 8
Dashboard: http://127.0.0.1:36521/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:41574
Local directory: /tmp/dask-worker-space/worker-k52s8uk1

Worker: 15
Comm: tcp://127.0.0.1:42776 Total threads: 8
Dashboard: http://127.0.0.1:36274/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:44074
Local directory: /tmp/dask-worker-space/worker-wolnue2f

read plotting information from a csv file¶

In [6]:
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Out[6]:
Value Inputs Equation Zone Plot Colourmap MinMax Unit Oldname Unnamed: 10
IceClim calc.IceClim_load(data,nc_outputpath) ALL IceClim Spectral (0,5) m M-4

Computation starts here¶

Each computation consists of

  1. Load NEMO data set
  2. Zoom data set
  3. Compute (or load computed data set)
  4. Save
  5. Plot
  6. Close
In [7]:
%%time
import os
calcswitch=os.environ.get('calc', 'True') 
lazy=os.environ.get('lazy','True' )
loaddata=((df.Inputs != '').any()) 
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0 
data

calcswitch= True df.Inputs != nothing False lazy= 
CPU times: user 359 µs, sys: 0 ns, total: 359 µs
Wall time: 359 µs
Out[7]:
0
In [8]:
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
            )

#calc= True
#save= False
#plot= True
Value='IceClim'
Zone='ALL'
Plot='IceClim'
cmap='Spectral'
clabel='m'
clim= (0, 5)
outputpath='../results/SEDNA_DELTA_MONITOR/'
nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/'
filename='SEDNA_IceClim_ALL_IceClim'
#3 Start computing 
dtaa= calc.IceClim_load(data,nc_outputpath)
start saving data
filename= ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_maps_ALL_IceConce/t_*/x_*/y_*.nc

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File <timed eval>:1, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py:64, in auto(df, val, savefig, daskreport, outputpath, file_exp)
     62 #print('count:',data.count())
     63 with performance_report(filename=daskreport+"_calc_"+step.Value+".html"):
---> 64     data=eval(command)
     65     #print('persist ')
     66     #data=data.persist()
     67 #print(data.count())
     68 #print('nbytes:',data.nbytes)
     69 #print('count:',data.count())
     70 optimize_dataset(data)

File <string>:1, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/calc.py:231, in IceClim_load(data, nc_outputpath)
    229 import xarray as xr
    230 filename='SEDNA_maps_ALL_IceConce'
--> 231 ds=save.load_data(plot='map',path=nc_outputpath,filename=filename)
    232 filename='SEDNA_maps_ALL_IceThickness'
    233 ds['sivolu']=save.load_data(plot='map',path=nc_outputpath,filename=filename).sivolu

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py:38, in load_data(plot, path, filename)
     36     data=load_twoD(path,filename,nested=False)    
     37 else:
---> 38     data=load_twoD(path,filename)        
     39 print('load computed data completed')    
     40 return data

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py:47, in load_twoD(path, filename, nested)
     45 filename=filesave+'/t_*/x_*/y_*.nc' if nested else filesave+'/t_*.nc'
     46 print ('filename=',filename)
---> 47 return xr.open_mfdataset(filename,parallel=True
     48               ,compat='override'
     49               ,data_vars='minimal'
     50               ,concat_dim=('x','y','t')
     51               ,combine='by_coords' #param_xios
     52               ,coords='minimal')

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:952, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
    947     ids, paths = (
    948         list(combined_ids_paths.keys()),
    949         list(combined_ids_paths.values()),
    950     )
    951 elif combine == "by_coords" and concat_dim is not None:
--> 952     raise ValueError(
    953         "When combine='by_coords', passing a value for `concat_dim` has no "
    954         "effect. To manually combine along a specific dimension you should "
    955         "instead specify combine='nested' along with a value for `concat_dim`.",
    956     )
    958 open_kwargs = dict(engine=engine, chunks=chunks or {}, **kwargs)
    960 if parallel:

ValueError: When combine='by_coords', passing a value for `concat_dim` has no effect. To manually combine along a specific dimension you should instead specify combine='nested' along with a value for `concat_dim`.
In [ ]: