(depth float64-> float32 included)
#import xarray as xr
#import pandas as pd
%matplotlib inline
#from IPython import display
import socket
#import sys
#import os
host = socket.getfqdn()
#sys.path.append(os.path.abspath(".."))
from core import load , save #, zoom, compute, plots, monitor,save
client, cluster=load.daskcluster(host,local='True')
using host= irene8002.c-irene.tgcc.ccc.cea.fr starting dask cluster on local= True workers 16 10000000000 False not local in tgcc c-irene.tgcc local FORCED tgcc local cluster starting
import importlib importlib.reload(load) importlib.reload(zoom) importlib.reload(compute) importlib.reload(plots) importlib.reload(monitor) importlib.reload(save)
%%time
import intake
import intake_xarray
import xarray as xr
if 'workbook' in host:
paramfile='../data/m*.nc'
catalog_url="../lib/CREG_home.yaml"
lonlat='../data/CREG025.L75-ERA01_y2001m10.5d_icemod.nc'
month="2"
paramkeep=xr.open_mfdataset(paramfile)
root_path='../data/tests'
paramkeep=paramkeep.isel(t=0)
cat = intake.open_catalog(catalog_url)
chunky=load.load_data_xios(cat,filename='gridT-2D',items='ssh',month=month).nav_lon.chunks[0]
else:
paramfile='/ccc/work/cont003/gen7420/talandel/CONFIGS/SEDNA/MESH/SEDNA_mesh_mask_Tgt_20210423_tsh10m_L1.nc'
catalog_url='../lib/SEDNA_ALPHA_MONITOR.yaml'
#lonlat='/ccc/work/cont003/gen7420/talandel/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20200808_tsh10m_L1.nc'
lonlat='/ccc/work/cont003/gen7420/talandel/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1.nc'
month='22'
year='*20040616'
path='/ccc/scratch/cont003/gen7420/talandel/ONGOING-RUNS/SEDNA-ALPHA-XIOS.22'
paramkeep=xr.open_dataset(paramfile)
root_path='/ccc/work/cont003/gen7420/odakatin/CONFIGS/SEDNA/SEDNA-I/SEDNA_Domain_cfg_Tgt_20210423_tsh10m_L1'
paramkeep=paramkeep.isel(time_counter=0).drop_vars('time_counter').reset_index( [ 'nav_lev'], drop = True).rename({'nav_lev': 'z'}).drop_vars(('nav_lon','nav_lat'))
cat = intake.open_catalog(catalog_url)
xarray_default_kwargs={'preprocess': load.prep
,'parallel': True
,'compat': 'override'
,'data_vars': 'minimal'
#,'use_cftime':True
#,'decode_cf': True
,'coords': 'minimal' }
#
filename='gridT-2D'
year='*'
xarray_kwargs={**xarray_default_kwargs}
#print('using load_data_xios reading ',filename)
desc=cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).describe
#print('using load_data_xios reading ',desc)
chunky = cat.data_xios(file=filename,month=month,year=year ,xarray_kwargs=xarray_kwargs).to_dask().nav_lon.chunks[0]
prefix=root_path+'/param_f32'
navlonlat=xr.open_dataset(lonlat)
CPU times: user 15.3 s, sys: 9.56 s, total: 24.8 s Wall time: 1min 49s
%%time
import numpy as np
param=xr.merge(\
[paramkeep.e3v_0, paramkeep.e1v])
param['nav_lon']=navlonlat.nav_lon[:,:]
param['nav_lat']=navlonlat.nav_lat[:,:]
param['mask']=xr.where(paramkeep.tmask==1, True, False)
param['depth']=paramkeep.gdept_0.astype(np.float32)
param['e3t']=paramkeep.e3t_0
param['e1te2t']=(paramkeep.e1t * paramkeep.e2t)
#param['sume1te2t']=param.e1te2t.sum()
param['mask2d']=param.mask[0,:,:]
param.coords['z']=range(1,(param.dims['z']+1))
param.coords['y']=range(1,(param.dims['y']+1))
param.coords['x']=range(1,(param.dims['x']+1))
param=param.chunk({'y': chunky}).unify_chunks()
param
CPU times: user 2min 52s, sys: 1min 15s, total: 4min 7s Wall time: 3min 47s
<xarray.Dataset> Dimensions: (x: 6560, y: 6540, z: 150) Coordinates: * z (z) int64 1 2 3 4 5 6 7 8 9 ... 142 143 144 145 146 147 148 149 150 * y (y) int64 1 2 3 4 5 6 7 8 ... 6534 6535 6536 6537 6538 6539 6540 * x (x) int64 1 2 3 4 5 6 7 8 ... 6554 6555 6556 6557 6558 6559 6560 Data variables: e3v_0 (z, y, x) float64 dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> e1v (y, x) float64 dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lon (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> nav_lat (y, x) float32 dask.array<chunksize=(13, 6560), meta=np.ndarray> mask (z, y, x) bool dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> depth (z, y, x) float32 dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> e3t (z, y, x) float64 dask.array<chunksize=(150, 13, 6560), meta=np.ndarray> e1te2t (y, x) float64 dask.array<chunksize=(13, 6560), meta=np.ndarray> mask2d (y, x) bool dask.array<chunksize=(13, 6560), meta=np.ndarray>
array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150])
array([ 1, 2, 3, ..., 6538, 6539, 6540])
array([ 1, 2, 3, ..., 6558, 6559, 6560])
|
|
|
|
|
|
|
|
|
%%time
param=param.persist()
CPU times: user 38.4 s, sys: 44 s, total: 1min 22s Wall time: 1min 18s
%%time
!rm -rf $prefix
pathcreated=save.to_mfnetcdf(param,prefix=prefix, nested=False)
CPU times: user 2min 22s, sys: 2min 4s, total: 4min 27s Wall time: 12min 53s
files=!find $prefix |grep '.nc'
param.nav_lon.plot()
<matplotlib.collections.QuadMesh at 0x2adbde827c50>