(depth float64-> float32 included)
%matplotlib inline
import socket
host = socket.getfqdn()
#sys.path.append(os.path.abspath(".."))
from core import load , save #, zoom, compute, plots, monitor,save
client, cluster=load.daskcluster(host,local='True')
client
using host= irene8002.c-irene.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False not local in tgcc c-irene.tgcc local FORCED tgcc local cluster starting
Client
|
Cluster
|
%%time
import intake
import intake_xarray
import xarray as xr
if 'workbook' in host:
paramfile='../data/m*.nc'
catalog_url="../lib/CREG_home.yaml"
lonlat='../data/CREG025.L75-ERA01_y2001m10.5d_icemod.nc'
month="2"
year="2001"
paramkeep=xr.open_mfdataset(paramfile)
root_path='../data/tests'
paramkeep=paramkeep.isel(t=0)
cat = intake.open_catalog(catalog_url)
chunky=load.load_data_xios(cat,filename='gridT-2D',items='ssh',month=month,year=year).nav_lon.chunks[0]
else:
paramfile='/ccc/work/cont003/gen7420/talandel/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1.nc'
catalog_url='../lib/SEDNA_ALPHA_MONITOR.yaml'
#lonlat='/ccc/work/cont003/gen7420/talandel/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20200808_tsh10m_L1.nc'
lonlat='/ccc/work/cont003/gen7420/talandel/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1.nc'
month='22'
year='*20040616'
path='/ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22'
paramkeep=xr.open_dataset(paramfile)
root_path='/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1'
paramkeep=paramkeep.isel(time_counter=0).drop_vars('time_counter').reset_index( [ 'nav_lev'], drop = True).rename({'nav_lev': 'z'}).drop_vars(('nav_lon','nav_lat'))
cat = intake.open_catalog(catalog_url)
xarray_default_kwargs={'preprocess': load.prep
,'parallel': True
,'compat': 'override'
,'data_vars': 'minimal'
#,'use_cftime':True
#,'decode_cf': True
,'coords': 'minimal' }
#
filename='gridT-2D'
year='*'
xarray_kwargs={**xarray_default_kwargs}
#print('using load_data_xios reading ',filename)
desc=cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).describe
#print('using load_data_xios reading ',desc)
chunky = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask().nav_lon.chunks[0]
prefix=root_path+'/param_f32'
navlonlat=xr.open_dataset(lonlat)
CPU times: user 20.5 s, sys: 18.6 s, total: 39.1 s Wall time: 2min 31s
%%time
import numpy as np
param=xr.merge(\
[paramkeep.e3v_0, paramkeep.e1v])
param['nav_lon']=navlonlat.nav_lon[:,:]
param['nav_lat']=navlonlat.nav_lat[:,:]
param['mask']=xr.where(paramkeep.tmask==1, True, False)
param['depth']=paramkeep.gdept_0.astype(np.float32)
param['e3t']=paramkeep.e3t_0
param['e1te2t']=(paramkeep.e1t * paramkeep.e2t)
#param['sume1te2t']=param.e1te2t.sum()
param['mask2d']=param.mask[0,:,:]
param.coords['z']=range(1,(param.dims['z']+1))
param.coords['y']=range(1,(param.dims['y']+1))
param.coords['x']=range(1,(param.dims['x']+1))
param=param.chunk({'y': chunky}).unify_chunks()
param
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) <timed exec> in <module> ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in chunk(self, chunks, name_prefix, token, lock) 1870 variables = { 1871 k: _maybe_chunk(k, v, chunks, token, lock, name_prefix) -> 1872 for k, v in self.variables.items() 1873 } 1874 return self._replace(variables) ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in <dictcomp>(.0) 1870 variables = { 1871 k: _maybe_chunk(k, v, chunks, token, lock, name_prefix) -> 1872 for k, v in self.variables.items() 1873 } 1874 return self._replace(variables) ~/monitor/lib/python3.7/site-packages/xarray/core/dataset.py in _maybe_chunk(name, var, chunks, token, lock, name_prefix, overwrite_encoded_chunks) 379 token2 = tokenize(name, token if token else var._data, chunks) 380 name2 = f"{name_prefix}{name}-{token2}" --> 381 var = var.chunk(chunks, name=name2, lock=lock) 382 383 if overwrite_encoded_chunks and var.chunks is not None: ~/monitor/lib/python3.7/site-packages/xarray/core/variable.py in chunk(self, chunks, name, lock) 1051 chunks = tuple(chunks.get(n, s) for n, s in enumerate(self.shape)) 1052 -> 1053 data = da.from_array(data, chunks, name=name, lock=lock, **kwargs) 1054 1055 return type(self)(self.dims, data, self._attrs, self._encoding, fastpath=True) ~/monitor/lib/python3.7/site-packages/dask/array/core.py in from_array(x, chunks, name, lock, asarray, fancy, getitem, meta, inline_array) 3053 3054 chunks = normalize_chunks( -> 3055 chunks, x.shape, dtype=x.dtype, previous_chunks=previous_chunks 3056 ) 3057 ~/monitor/lib/python3.7/site-packages/dask/array/core.py in normalize_chunks(chunks, shape, limit, dtype, previous_chunks) 2726 raise ValueError( 2727 "Chunks do not add up to shape. " -> 2728 "Got chunks=%s, shape=%s" % (chunks, shape) 2729 ) 2730 ValueError: Chunks do not add up to shape. Got chunks=((150,), (13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12), (6560,)), shape=(150, 6540, 6560)
#%%time
#param=param.persist()
%%time
!rm -rf $prefix
pathcreated=save.to_mfnetcdf(param,prefix=prefix, nested=False)
debug filename list /ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1/param_f32.nc CPU times: user 6min 43s, sys: 2min 5s, total: 8min 49s Wall time: 8min 50s
files=!find $prefix |grep '.nc'
param.nav_lon.plot()
<matplotlib.collections.QuadMesh at 0x2ac5583b9210>