Dask Performance Report

Select different tabs on the top for additional information

Duration: 46.00 s

Tasks Information

  • number of tasks: 5542
  • compute time: 365.10 s
  • transfer time: 25.37 s

Scheduler Information

  • Address: tcp://127.0.0.1:36549
  • Workers: 12
  • Threads: 48
  • Memory: 2.86 TiB
  • Dask Version: 2022.7.1
  • Dask.Distributed Version: 2022.7.1

Calling Code

def auto(df,val,savefig,daskreport,outputpath='../results/',file_exp='SEDNA'):
    # val=val.persist()
    saveswitch=os.environ.get('save', 'True')
    plotswitch=os.environ.get('plot', 'True')
    calcswitch=os.environ.get('calc', 'True')
    nc_outputpath=re.sub(r'result', 'nc_result', outputpath)
    os.makedirs(nc_outputpath, exist_ok=True)
    print('#calc=',calcswitch )  
    print('#save=',saveswitch )  
    print('#plot=',plotswitch )  
    for step in df.itertuples():
    # 1. Create data set

        data=val
        Value=step.Value
        Zone=step.Zone
        Plot=step.Plot
        cmap=step.Colourmap 
        clim=eval(step.MinMax)
        clabel=step.Unit 
        filename=file_exp+'_'+Plot+'_'+Zone+'_'+Value
        filename=filename.replace(' ', '_')
        print('Value=\'' + Value + '\'' )  
        print('Zone=\'' + Zone + '\'' )  
        print('Plot=\'' + Plot + '\'' )  
        print('cmap=\'' + cmap + '\'' )  
        print('clabel=\'' + clabel + '\'' )  
        print('clim=',clim )  
        print('outputpath=\'' + outputpath + '\'' )  
        print('nc_outputpath=\'' + nc_outputpath + '\'' )  
        print('filename=\'' + filename + '\'' )  
        #print(data)
    # 2. Zoom Data set
        if data != 0 :
            data=optimize_dataset(data)
            print('data=monitor.optimize_dataset(data)')
            if step.Zone != 'ALL' :
                print('#2 Zooming Data')
                command='zoom.'+Zone +'(data)'
                print('data=',command)
                with performance_report(filename=daskreport+"_zoom_"+Value+".html"):
                    data=eval(command)
                    data=optimize_dataset(data)
                    print('data=monitor.optimize_dataset(data)')
                display(data)
    # 3. Compute  (or load computed results)
        command=step.Equation
        if calcswitch=='True': 
            print('#3 Start computing ')
            print('data=',command )
            print('monitor.optimize_dataset(data)') 
            #print('count:',data.count())
            with performance_report(filename=daskreport+"_calc_"+step.Value+".html"):
                data=eval(command)
                #print('persist ')
                #data=data.persist()
                print('add optimise here once otimise can recognise')
                #data=optimize_dataset(data)                
                #print(data.count())
                #print('nbytes:',data.nbytes)
                #print('count:',data.count())
        else:
            print('#3 no computing , loading starts')
            print('data=save.load_data(plot=Plot,path=nc_outputpath,filename=filename)' )
            with performance_report(filename=daskreport+"_calc_"+step.Value+".html"):
                data=save.load_data(plot=Plot,path=nc_outputpath,filename=filename)
            #saveswitch=False

        display(data)
                
    # 4. Save
        if saveswitch=='True': 
            print('#4 Saving ',filename )  
            print('data=save.datas(data,plot=Plot,path=nc_outputpath,filename=filename)' )
            with performance_report(filename=daskreport+"_save_"+step.Value+".html"):
                save.datas(data,plot=Plot,path=nc_outputpath,filename=filename)                
    # 5. Plot       
        if plotswitch=='True': 
            command='plots.'+step.Plot \
                +'(data,path=outputpath,filename=filename,save=savefig' \
                +',cmap=cmap,clim=clim,clabel=clabel)'    
            print('#5 Plotting')  
            print('filename=',command )  
            with performance_report(filename=daskreport+"_plot_"+step.Value+".html"):
                filename=eval(command )
            print(filename,'created')
            display(IFrame(filename, width=1000, height=500))
    return

        
Summary
Task Stream
System
Scheduler Logs
Worker Profile (compute)
Worker Profile (administrative)
Scheduler Profile (administrative)
Bandwidth (Workers)
Bandwidth (Types)