In [1]:
%matplotlib inline
import pandas as pd
import socket
host = socket.getfqdn()

from core import  load, zoom, calc, save,plots,monitor
In [2]:
#reload funcs after updating ./core/*.py
import importlib
importlib.reload(load)
importlib.reload(zoom)
importlib.reload(calc)
importlib.reload(save)
importlib.reload(plots)
importlib.reload(monitor)
Out[2]:
<module 'core.monitor' from '/ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py'>
In [3]:
# 'month':  = 'JOBID' almost month but not really, 

# If you submit the job with job scheduler, above

#below are list of enviroment variable one can pass
#%env local='2"
# local : if True  run dask local cluster, if not true, put number of workers
# setted in the 'local'
# if no 'local ' given, local will be setted automatically to 'True'
#%env ychunk='2'
#%env tchunk='2'
# controls chunk. 'False' sets no modification from original netcdf file's chunk.  
# ychunk=10 will group the original netcdf file to 10 by 10 
# tchunk=1 will chunk the time coordinate one by one
#%env control=FWC_SSH 
# name of control file to be used for computation/plots/save/ 
#%env file_exp= 
# 'file_exp': Which 'experiment' name is it? 
#.    this corresopnds to intake catalog name without path and .yaml
#%env year=
# for Validation, this correspoinds to path/year/month 's year
# for monitoring, this corresponids to 'date'  having * means do all files in the monitoring directory
# setting it as *0[0-9] &*1[0-9]& *[2-3][0-9], the job can be separated in three lots.
#%env month=
# for monitoring  this corresponds to file path path-XIOS.{month}/
#
#%env save=   proceed saving?   True or False  , Default is setted as True 
#%env plot=   proceed plotting?  True or False , Default is setted as True 
#%env calc=   proceed computation? or just load computed result? True or False , Default is setted as True 
#%env save=False
#%env lazy=False
In [4]:
%env file_exp=SEDNA_DELTA_MONITOR
%env year=2012
%env month=02
%env ychunk=10
%env save=False
%env plot=True
%env calc=True
%env lazy=
#False
%env control=M_IceClim
#M_Sectiony ok with ychunk=False local=True  lazy=False 

env: file_exp=SEDNA_DELTA_MONITOR
env: year=2012
env: month=02
env: ychunk=10
env: save=False
env: plot=True
env: calc=True
env: lazy=
env: control=M_IceClim
In [5]:
%%time
# 'savefig': Do we save output in html? or not. keep it true. 
savefig=True
client,cluster,control,catalog_url,month,year,daskreport,outputpath = load.set_control(host)
!mkdir -p $outputpath
!mkdir -p $daskreport
client

local True
using host= irene5625.c-irene.mg1.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16
10000000000
False
rome local cluster starting
This code is running on  irene5625.c-irene.mg1.tgcc.ccc.cea.fr using  SEDNA_DELTA_MONITOR file experiment, read from  ../lib/SEDNA_DELTA_MONITOR.yaml  on year= 2012  on month= 02  outputpath= ../results/SEDNA_DELTA_MONITOR/ daskreport= ../results/dask/6414198irene5625.c-irene.mg1.tgcc.ccc.cea.fr_SEDNA_DELTA_MONITOR_02M_IceClim/_lazy
CPU times: user 516 ms, sys: 118 ms, total: 634 ms
Wall time: 20.3 s
Out[5]:

Client

Client-df976a03-1349-11ed-a83a-080038b93a3b

Connection method: Cluster object Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Cluster Info

LocalCluster

a15c8675

Dashboard: http://127.0.0.1:8787/status Workers: 16
Total threads: 128 Total memory: 251.06 GiB
Status: running Using processes: True

Scheduler Info

Scheduler

Scheduler-d9d274c5-29f4-4c99-99d1-506d57cb57f2

Comm: tcp://127.0.0.1:44356 Workers: 16
Dashboard: http://127.0.0.1:8787/status Total threads: 128
Started: Just now Total memory: 251.06 GiB

Workers

Worker: 0
Comm: tcp://127.0.0.1:38603 Total threads: 8
Dashboard: http://127.0.0.1:41023/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:34115
Local directory: /tmp/dask-worker-space/worker-mnj9m4j8

Worker: 1
Comm: tcp://127.0.0.1:46713 Total threads: 8
Dashboard: http://127.0.0.1:46777/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:43259
Local directory: /tmp/dask-worker-space/worker-y_zywu8c

Worker: 2
Comm: tcp://127.0.0.1:33656 Total threads: 8
Dashboard: http://127.0.0.1:40662/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:40398
Local directory: /tmp/dask-worker-space/worker-5szbvgi8

Worker: 3
Comm: tcp://127.0.0.1:37123 Total threads: 8
Dashboard: http://127.0.0.1:44807/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:37094
Local directory: /tmp/dask-worker-space/worker-759le1xf

Worker: 4
Comm: tcp://127.0.0.1:38085 Total threads: 8
Dashboard: http://127.0.0.1:41192/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:46871
Local directory: /tmp/dask-worker-space/worker-suaxmcnc

Worker: 5
Comm: tcp://127.0.0.1:37856 Total threads: 8
Dashboard: http://127.0.0.1:35497/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:41536
Local directory: /tmp/dask-worker-space/worker-puh5cfr2

Worker: 6
Comm: tcp://127.0.0.1:34367 Total threads: 8
Dashboard: http://127.0.0.1:37427/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:40602
Local directory: /tmp/dask-worker-space/worker-3eavm0it

Worker: 7
Comm: tcp://127.0.0.1:40482 Total threads: 8
Dashboard: http://127.0.0.1:42475/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:46220
Local directory: /tmp/dask-worker-space/worker-35fnbir6

Worker: 8
Comm: tcp://127.0.0.1:42152 Total threads: 8
Dashboard: http://127.0.0.1:40816/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:37716
Local directory: /tmp/dask-worker-space/worker-unl4seoo

Worker: 9
Comm: tcp://127.0.0.1:36949 Total threads: 8
Dashboard: http://127.0.0.1:44565/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:38830
Local directory: /tmp/dask-worker-space/worker-547po3w7

Worker: 10
Comm: tcp://127.0.0.1:35997 Total threads: 8
Dashboard: http://127.0.0.1:46392/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:44004
Local directory: /tmp/dask-worker-space/worker-apatkw2y

Worker: 11
Comm: tcp://127.0.0.1:36791 Total threads: 8
Dashboard: http://127.0.0.1:37671/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:37674
Local directory: /tmp/dask-worker-space/worker-swvy1xu9

Worker: 12
Comm: tcp://127.0.0.1:45904 Total threads: 8
Dashboard: http://127.0.0.1:38699/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:37060
Local directory: /tmp/dask-worker-space/worker-6wp7ir1k

Worker: 13
Comm: tcp://127.0.0.1:35534 Total threads: 8
Dashboard: http://127.0.0.1:37659/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:38800
Local directory: /tmp/dask-worker-space/worker-fgfwrr2a

Worker: 14
Comm: tcp://127.0.0.1:40734 Total threads: 8
Dashboard: http://127.0.0.1:46704/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:38219
Local directory: /tmp/dask-worker-space/worker-ikgjhxjn

Worker: 15
Comm: tcp://127.0.0.1:43450 Total threads: 8
Dashboard: http://127.0.0.1:38806/status Memory: 15.69 GiB
Nanny: tcp://127.0.0.1:38387
Local directory: /tmp/dask-worker-space/worker-z6exal77

read plotting information from a csv file¶

In [6]:
df=load.controlfile(control)
#Take out 'later' tagged computations
#df=df[~df['Value'].str.contains('later')]
df
Out[6]:
Value Inputs Equation Zone Plot Colourmap MinMax Unit Oldname Unnamed: 10
IceClim calc.IceClim_load(data,nc_outputpath) ALL IceClim Spectral (0,5) m M-4

Computation starts here¶

Each computation consists of

  1. Load NEMO data set
  2. Zoom data set
  3. Compute (or load computed data set)
  4. Save
  5. Plot
  6. Close
In [7]:
%%time
import os
calcswitch=os.environ.get('calc', 'True') 
lazy=os.environ.get('lazy','True' )
loaddata=((df.Inputs != '').any()) 
print('calcswitch=',calcswitch,'df.Inputs != nothing',loaddata, 'lazy=',lazy)
data = load.datas(catalog_url,df.Inputs,month,year,daskreport,lazy=lazy) if ((calcswitch=='True' )*loaddata) else 0 
data

calcswitch= True df.Inputs != nothing False lazy= 
CPU times: user 408 µs, sys: 61 µs, total: 469 µs
Wall time: 419 µs
Out[7]:
0
In [8]:
%%time
monitor.auto(df,data,savefig,daskreport,outputpath,file_exp='SEDNA'
            )

#calc= True
#save= False
#plot= True
Value='IceClim'
Zone='ALL'
Plot='IceClim'
cmap='Spectral'
clabel='m'
clim= (0, 5)
outputpath='../results/SEDNA_DELTA_MONITOR/'
nc_outputpath='../nc_results/SEDNA_DELTA_MONITOR/'
filename='SEDNA_IceClim_ALL_IceClim'
#3 Start computing 
dtaa= calc.IceClim_load(data,nc_outputpath)
start saving data
filename= ../nc_results/SEDNA_DELTA_MONITOR/SEDNA_maps_ALL_IceConce/t_*/x_*/y_*.nc

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File <timed eval>:1, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/monitor.py:64, in auto(df, val, savefig, daskreport, outputpath, file_exp)
     62 #print('count:',data.count())
     63 with performance_report(filename=daskreport+"_calc_"+step.Value+".html"):
---> 64     data=eval(command)
     65     #print('persist ')
     66     #data=data.persist()
     67 #print(data.count())
     68 #print('nbytes:',data.nbytes)
     69 #print('count:',data.count())
     70 optimize_dataset(data)

File <string>:1, in <module>

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/calc.py:231, in IceClim_load(data, nc_outputpath)
    229 import xarray as xr
    230 filename='SEDNA_maps_ALL_IceConce'
--> 231 ds=save.load_data(plot='map',path=nc_outputpath,filename=filename)
    232 filename='SEDNA_maps_ALL_IceThickness'
    233 ds['sivolu']=save.load_data(plot='map',path=nc_outputpath,filename=filename).sivolu

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py:38, in load_data(plot, path, filename)
     36     data=load_twoD(path,filename,nested=False)    
     37 else:
---> 38     data=load_twoD(path,filename)        
     39 print('load computed data completed')    
     40 return data

File /ccc/work/cont003/gen7420/odakatin/monitor-sedna/notebook/core/save.py:47, in load_twoD(path, filename, nested)
     45 filename=filesave+'/t_*/x_*/y_*.nc' if nested else filesave+'/t_*.nc'
     46 print ('filename=',filename)
---> 47 return xr.open_mfdataset(filename,parallel=True
     48               ,compat='override'
     49               ,data_vars='minimal'
     50               ,concat_dim=('x','y','t')
     51               ,combine='by_coords' #param_xios
     52               ,coords='minimal')

File /ccc/cont003/home/ra5563/ra5563/monitor/lib/python3.10/site-packages/xarray/backends/api.py:952, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
    947     ids, paths = (
    948         list(combined_ids_paths.keys()),
    949         list(combined_ids_paths.values()),
    950     )
    951 elif combine == "by_coords" and concat_dim is not None:
--> 952     raise ValueError(
    953         "When combine='by_coords', passing a value for `concat_dim` has no "
    954         "effect. To manually combine along a specific dimension you should "
    955         "instead specify combine='nested' along with a value for `concat_dim`.",
    956     )
    958 open_kwargs = dict(engine=engine, chunks=chunks or {}, **kwargs)
    960 if parallel:

ValueError: When combine='by_coords', passing a value for `concat_dim` has no effect. To manually combine along a specific dimension you should instead specify combine='nested' along with a value for `concat_dim`.
In [ ]: