U
    /e4                    @  s,  d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlZd dlmZmZ d d	lmZ d d
lmZmZ d dlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 d dl5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZBmCZCmDZD d dlEmFZFmGZG d dlHmIZImJZJmKZK d dlLmMZMmNZNmOZO d dlPmQZQ d dlRZRd dlRmSZS d dlTmUZUmVZVmWZWmXZXmYZYmZZZ d dl[m\Z\ d dl]m^Z^ d dl_m`Z`maZambZbmcZc d dldmeZe d dlfmgZgmhZhmiZimjZjmkZk d dllmmZm d dlnmoZo d dlpmqZqmrZr d d lsmtZt d d!lsmqZu d d"lsmvZw d d#lxmyZy d d$lzm{Z{ d d%l|m}Z}m~Z~ eRjSd&rd d'lmZ ndZed(ZeeZeFeGejejed)d)d*d+d,Ze?ejejed)d-d.Zd/d0d1d2d3d4d5d6d7d8d9g
d:Zej d; Zd<d=d>d?ZG d@dA dAe`ZG dBdC dCe`ZG dDdE dEZG dFdG dGe`eZG dHdI dIe`eZG dJdK dKe`ZG dLdM dMe`ZG dNdO dOe`ZG dPdQ dQe`ZG dRdS dSe`ZG dTdU dUe`ZG dVdW dWe`ZG dXdY dYe`ZG dZd[ d[e`ZG d\d] d]e`ZG d^d_ d_e`ZG d`da dae`ZG dbdc dce`ZG ddde dee`ZG dfdg dge`ZddidjZG dkdl dle`ZG dmdn dne`ZG dodp dpe`ZG dqdr dre`ZG dsdt dte`ZG dudv dve`ZG dwdx dxe`ZG dydz dze`Zd{d{d|d}d~ddddZehjdk rPdeiZndeiZG dd dZe~dd Ze~dd Ze~dd Ze~dd Ze~dd Ze~dd Ze~dd Ze~dd Ze~dd Ze~dd Ze~dd ZeIdddZe~dd Ze~dd Ze~dd Ze~dd ZdS )    )annotationsN)OrderedDictdefaultdict)Iterable)datetime)Number)AnyTypeVar)valuewithout_property_validation)curdoc)columnrow)AdaptiveTickerArrowBasicTickerBoxSelectToolBoxZoomToolCDSViewColorBarColumnDataSourceCustomJSHoverDataRange1dFactorRangeGroupFilter	HoverToolHTMLTemplateFormatterNumberFormatterNumeralTickFormatterOpenURLPanToolRange1d	ResetToolTabsTapToolTitleVeeHeadWheelZoomTool)	DataTableTableColumn)Div)	Viridis11)figure)Theme)cumsumfactor_cmaplinear_cmapstack)EnvironmentFileSystemLoader)currypipevalmap)concatgroupbymap)escape)config)format_bytesformat_timefuncname	key_splitparse_bytesparse_timedelta)Status)add_periodic_callback)DashboardComponentProfileServerProfileTimePlotSystemMonitor)TabPanel)_DATATABLE_STYLESHEETS_KWARGSBOKEH_VERSION	PROFILING	transposeupdate)GraphLayout)GroupTiming)color_ofprogress_quads)TaskStreamPluginrP   )colors)time)	Scheduler)Log
log_errorsz!distributed.dashboard.export-tool)
ExportToolTz..httpZ	templates)loaderz
theme.yaml)filenamei                      @         i   )baseZ	mantissas	   zstatics/images/numpy.pngzstatics/images/pandas.pngzstatics/images/python.png)numpyZpandasbuiltinsc                   @  s,   e Zd ZdZedd Zeedd ZdS )	OccupancyOccupancy (in time) per workerc              	   K  s   || _ tddgddgddgddgddgdd	gddgd
| _tf dddddd|| _| jj| jdddddd}d |_d| jj_d| jj	_
d| jj_
d| jj_ttddd}t }d|_d|_| j|| d S )Nr   ab        皙?r^   r_   redblue)	occupancyworkerxymscolorescaped_workerrk    abover   2   )titletoolstoolbar_locationx_axis_typemin_border_bottomru   rw   rv   ?rx   )sourceru   widthrv   heightrx   F"./info/worker/@escaped_worker.htmlurlcallbackz@worker : @occupancy s.follow_mouse)	schedulerr   r   r,   rootrectnonselection_glyphxaxisminor_tick_line_alphayaxisvisibleygridx_rangestartr$   r   r   tooltipspoint_policy	add_tools)selfr   kwargsr   taphover r   N/tmp/pip-unpacked-wheel-g426oqom/distributed/dashboard/components/scheduler.py__init__|   sL         



zOccupancy.__init__c           
      C  s
  | j j }ttt|}dd |D }dd |D }dd |D }t|}g }|D ]>}|| j jkrr|d qV|| j j	kr|d qV|d qV|rdt
| d	t
|| j j  | jj_n
d
| jj_|r|dd |D ||dd |D ||d}	t| j|	 d S )Nc                 S  s   g | ]
}|j qS r   )rs   .0wsr   r   r   
<listcomp>   s     z$Occupancy.update.<locals>.<listcomp>c                 S  s   g | ]}|d  qS )  r   r   Zoccr   r   r   r      s     c                 S  s   g | ]}|d  qS )  r   r   r   r   r   r      s     rq   greenrr   zOccupancy -- total time: z wall time: rk   c                 S  s   g | ]
}|j qS r   addressr   r   r   r   r      s     c                 S  s   g | ]}t |jqS r   r:   
url_escaper   r   r   r   r   r      s     )rs   rt   rw   rx   ry   ru   rv   )r   workersvalueslistrangelensumidleappend	saturatedr=   Ztotal_nthreadsr   r}   textrM   r   )
r   r   rv   rs   rw   ru   totalrx   r   resultr   r   r   rM      s6    


zOccupancy.updateN__name__
__module____qualname____doc__rX   r   r   rM   r   r   r   r   rk   y   s   
(rk   c                   @  s(   e Zd ZdZedd Zedd ZdS )ProcessingHistogram!How many tasks are on each workerc              	   K  s   d| _ || _tddgddgddgd| _tf dddd	d
|| _d| jj_d| jj_	d | j_
| jj| jddddddd d S )Nr   r^   r_   
   leftrighttopzTasks Processing (count)
processing	frequencyrz   r}   nameZy_axis_labelr~   Fr   r   r   deepskyblue      ?r   r   r   bottomr   rx   
fill_alpha)lastr   r   r   r,   r   r   r   r   r   r   quadr   r   r   r   r   r   r      s2    

zProcessingHistogram.__init__c                 C  sR   dd | j j D }tj|dd\}}| jj|d d |dd  |d d S )Nc                 S  s   g | ]}t |jqS r   r   r   r   r   r   r   r      s     z.ProcessingHistogram.update.<locals>.<listcomp>(   Zbinsr^   r   )r   r   r   np	histogramr   datarM   )r   Lcountsru   r   r   r   rM      s    zProcessingHistogram.updateNr   r   r   r   r   r      s
   
r   c                   @  s>   e Zd ZU dZded< ded< dd Zdddd	d
ddZdS )MemoryColora  Change the color of the memory bars from blue to orange when process memory goes
    above the ``target`` threshold and to red when the worker pauses.
    Workers in ``closing_gracefully`` state will also be orange.

    If ``target`` is disabled, change to orange on ``spill`` instead.
    If spilling is completely disabled, never turn orange.

    If pausing is disabled, change to red when passing the ``terminate`` threshold
    instead. If both pause and terminate are disabled, turn red when passing
    ``memory_limit``.

    Note
    ----
    A worker will start spilling when managed memory alone passes the target threshold.
    However, here we're switching to orange when the process memory goes beyond target,
    which is usually earlier.
    This is deliberate for the sake of simplicity and also because, when the process
    memory passes the spill threshold, it will keep spilling until it falls below the
    target threshold - so it's not completely wrong. Again, we don't want to track
    the hysteresis cycle of the spill system here for the sake of simplicity.

    In short, orange should be treated as "the worker *may* be spilling".
    floatorangerq   c                 C  sR   t jd}t jd}t jd}t|p.tj|p6tj| _t|pFtjd| _d S )Nz distributed.worker.memory.targetzdistributed.worker.memory.spillz#distributed.worker.memory.terminate      ?)daskr;   getminmathinfr   rq   )r   targetZspill	terminater   r   r   r     s
    zMemoryColor.__init__intrB   str)currentlimitstatusreturnc                 C  s>   |t jkrdS |sdS ||| j kr(dS ||| j kr:dS dS )Nrq   rr   r   )rB   Zrunningrq   r   )r   r   r   r   r   r   r   _memory_color  s    
zMemoryColor._memory_colorN)r   r   r   r   __annotations__r   r   r   r   r   r   r      s
   
	r   c                   @  s<   e Zd ZdZedddZddddZeed	d
 ZdS )ClusterMemoryz!Total memory usage on the clusterX  c                 K  sL  t |  t|  || _tdgd dgd dgd ddddgddddgdgd dgd dgd dgd dgd d
| _tf d	d
t|d ddd|| _| jj	| jddddddd}d |_
tf t| jjd _tdd| jjd _t| jj_d| jj_tdd| j_d| jj_d| jj_d | j_d| jj_tddd}| j| d S )Nr   r`   rr   greyr^   ffffff?皙?)
r   ru   rv   rx   alphaproc_memorymanagedunmanaged_oldunmanaged_recentspilledzBytes stored on clusterrz   r_   cluster_memoryr|   r}   r~   r   r   r   ru   rv   r   r   rx   r   )r   ru   rv   r   r   rx   r   0.0 bformatr   Fr   aZ  
                            <div>
                                <span style="font-size: 12px; font-weight: bold;">Process memory (RSS):</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@proc_memory{0.00 b}</span>
                            </div>
                            <div style="margin-left: 1em;">
                                <span style="font-size: 12px; font-weight: bold;">Managed:</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@managed{0.00 b}</span>
                            </div>
                            <div style="margin-left: 1em;">
                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (old):</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_old{0.00 b}</span>
                            </div>
                            <div style="margin-left: 1em;">
                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (recent):</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_recent{0.00 b}</span>
                            </div>
                            <div>
                                <span style="font-size: 12px; font-weight: bold;">Spilled to disk:</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@spilled{0.00 b}</span>
                            </div>
                            r   r   )rD   r   r   r   r   r   r,   r   r   r   r   r   
TICKS_1024axistickerr   r   	formatterXLABEL_ORIENTATIONmajor_label_orientationr   r!   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   +  sb    




	




zClusterMemory.__init__r   r   c                   sN    fdd j j D }|dddhs.td|kr:dS d|krFdS dS d S )Nc                   s*   h | ]"} j |jjt|d d|jdqS )memory_limitr   )r   r   r   )r   memoryprocessgetattrr   r   r   r   r   	<setcomp>z  s   
z6ClusterMemory._cluster_memory_color.<locals>.<setcomp>rq   r   rr   )r   r   r   issubsetAssertionError)r   rT   r   r
  r   _cluster_memory_colory  s    

	z#ClusterMemory._cluster_memory_colorc              	     s   t dd | jj D }| jj}|  }|j|j|j|j	g   fddt
 D |||dg|jgd |jgd |jgd |jgd |j	gd d}t||j|j	 }|| jj_dt|j }|j	r|d	t|j	 d
7 }|| jj_t| j| d S )Nc                 s  s   | ]}|j V  qd S Nr  r   r   r   r   	<genexpr>  s     z'ClusterMemory.update.<locals>.<genexpr>c                   s(   g | ] \}}t  d | |d  qS )Nr_   r   )r   iwr   r   r   r     s     z(ClusterMemory.update.<locals>.<listcomp>r   r`   )r   ru   rx   r   r   r   r   r   zBytes stored: z + z spilled to disk)r   r   r   r   r  r  r   r   r   r   	enumerater  maxr   r   endr<   r}   r   rM   r   )r   r   meminforx   r   x_endr}   r   r  r   rM     s0    







zClusterMemory.updateN)r   )	r   r   r   r   rX   r   r  r   rM   r   r   r   r   r   (  s   Mr   c                   @  s.   e Zd ZdZedddZeedd ZdS )	WorkersMemoryzMemory usage for single workersr   c                 K  s$  t |  t|  || _tg g g g g g g g g g g g d| _tf ddt|d ddd|| _| jj	| jdd	d
ddddd}d |_
tf t| jjd _tdd| jjd _t| jj_d| jj_tdd| j_d| jj_d| jj_d | j_ttddd}| j| tddd}| j| d S )N)r   ru   rv   rx   r   rt   ry   r   r   r   r   r   Bytes stored per workerrz   r_   workers_memoryr|   r   ru   rv   r   r   rx   r   r   )r   ru   rv   r   r   rx   r   
line_widthr   r   r   Fr   r   r   r   at  
                            <div>
                                <span style="font-size: 12px; font-weight: bold;">Worker:</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@worker</span>
                            </div>
                            <div>
                                <span style="font-size: 12px; font-weight: bold;">Process memory (RSS):</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@proc_memory{0.00 b}</span>
                            </div>
                            <div style="margin-left: 1em;">
                                <span style="font-size: 12px; font-weight: bold;">Managed:</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@managed{0.00 b}</span>
                            </div>
                            <div style="margin-left: 1em;">
                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (old):</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_old{0.00 b}</span>
                            </div>
                            <div style="margin-left: 1em;">
                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (recent):</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_recent{0.00 b}</span>
                            </div>
                            <div>
                                <span style="font-size: 12px; font-weight: bold;">Spilled to disk:</span>&nbsp;
                                <span style="font-size: 10px; font-family: Monaco, monospace;">@spilled{0.00 b}</span>
                            </div>
                            r   )rD   r   r   r   r   r   r,   r   r   r   r   r   r   r   r  r   r   r  r  r  r   r!   r   r   r   r   r   r$   r   r   r   )r   r   r   r   r   r   r   r   r   r   r     sj    







zWorkersMemory.__init__c                   s  ddddd}| j j }g  g }g }d}g }g }g }g }	g }
|D ]}|j}t|dd}t|||j|j }| |j||j	} |j
|j|j|jg7  | fdd	td
dD 7 }||||dg7 }||j ||j
 |	|j |
|j ||j qD ||ddddgt| |dd |D |dd |D |tt|||||||	||
||d} fdd| D }|| jj_t| j| d S )NzIterable[T]zlist[T])r  r   c                 S  s"   g }| D ]}|||||g7 }q|S r  r   )r  outiir   r   r   quadlist  s    z&WorkersMemory.update.<locals>.quadlistr   r  c                   s(   g | ] }t  d |  | d  qS )r_   r  r   r  r  r   r   r   )  s     z(WorkersMemory.update.<locals>.<listcomp>r"  r   r^   r   r   c                 s  s   | ]}|j V  qd S r  r   r   r   r   r   r  8  s     z'WorkersMemory.update.<locals>.<genexpr>c                 s  s   | ]}t |jV  qd S r  r   r   r   r   r   r  9  s     )r   ru   rx   r   rt   ry   rv   r   r   r   r   r   c                   s&   i | ]\}}|d d t | D qS )c                 S  s   g | ]\}}|r|qS r   r   )r   vir  r   r   r   r   B  s      z3WorkersMemory.update.<locals>.<dictcomp>.<listcomp>)zipr   kvr  r   r   
<dictcomp>B  s      z(WorkersMemory.update.<locals>.<dictcomp>)r   r   r   r  r	  r  r  r   r   r   r   r   r   r   r   r   itemsr   r   r  rM   r   )r   r!  r   ru   rx   Z	max_limitZ
procmemoryr   r   r   r   r   r  r   Zcolor_ir   r   r  r   rM     sZ    
zWorkersMemory.updateN)r   r   r   r   r   r   r    s   Ur  c                   @  s(   e Zd ZdZedd Zedd ZdS )WorkersMemoryHistogramzHistogram of memory usage, showing how many workers there are in each bucket of
    usage. Replaces the per-worker graph when there are >= 50 workers.
    c              	   K  s   d| _ || _tddgddgddgd| _tf dddd	d
|| _tdd| jjd _t	f t
| jj_t| jj_d| jj_d| jj_d | j_| jj| jddddddd d S )Nr   r^   r_   r   r   r  r  r   rz   r   r   r   Fr   r   r   r   r   r   )r   r   r   r   r,   r   r   r   r  r   r   r  r  r  r   r   r   r   r   r   r   r   r   r   M  s8    


zWorkersMemoryHistogram.__init__c                 C  sZ   t dd | jj D }t j|dd\}}|d d |dd  |d}t| j| d S )Nc                 S  s   g | ]}|j d  qS )r  metricsr   r   r   r   r   s  s     z1WorkersMemoryHistogram.update.<locals>.<listcomp>r   r   r   r^   r   )r   Zasarrayr   r   r   r   rM   r   )r   nbytesr   ru   dr   r   r   rM   p  s    zWorkersMemoryHistogram.updateNr   r   r   r   r   r+  H  s
   
"r+  c                   @  s.   e Zd ZdZedddZeedd ZdS )	WorkersTransferBytesz<Size of open data transfers from/to other workers per workerr   c              
   K  s"  || _ tg g g g g g d| _tf dtd dt|d ddd|| _| jjd	d
d	d ddd| jd | jjdddd ddd| jd tf t	| jj
d _tdd| jjd _t| jj_d| jj_tdd| j_d| jj_d| jj_d | j_ttddd}tdddgdd}| j|| d S )Nry   transfer_incoming_bytestransfer_outgoing_bytesrt   
y_incoming
y_outgoingBytes transferring: r   rz   r_   workers_transfer_bytesr|   r   r2  r4  r   rq   )r   rv   r   
line_colorr   r   
fill_colorr   r3  r5  rr   r   r   r   Fr   r   r   )ZWorkerz@worker)ZIncomingz @transfer_incoming_bytes{0.00 b})ZOutgoingz @transfer_outgoing_bytes{0.00 b}r   )r   r   )r   r   r   r,   r<   r   r   hbarr   r   r   r  r   r   r  r  r  r   r!   r   r   r   r   r   r$   r   r   r   )r   r   r   r   r   r   r   r   r   r   }  sp    





zWorkersTransferBytes.__init__c           
        s   | j j }d  fddtt|D } fddtt|D }dd |D }dd |D }dd |D }dd |D }|rtt|t|td	d
 |D }nd}|| jj_||||||d}	dt	t
| | jj_t| j|	 d S )Nrp   c                   s   g | ]}|d  |   qS g      ?r   r#  hr   r   r     s     z/WorkersTransferBytes.update.<locals>.<listcomp>c                   s   g | ]}|d  |   qS g      ?r   r#  r<  r   r   r     s     c                 S  s   g | ]}|j d  d qS )transferZincoming_bytesr,  r   r   r   r   r     s    c                 S  s   g | ]}|j d  d qS )r?  Zoutgoing_bytesr,  r   r   r   r   r     s    c                 S  s   g | ]
}|j qS r   r   r   r   r   r   r     s     c                 S  s   g | ]}t |qS r   )r:   r   )r   rt   r   r   r   r     s     c                 s  s   | ]}|j V  qd S r  r  r   r   r   r   r    s     z.WorkersTransferBytes.update.<locals>.<genexpr>r   r1  r6  )r   r   r   r   r   r  r   r   r  r<   r   r}   r   rM   r   )
r   wssr4  r5  r2  r3  r   Zescaped_workersZx_limitr   r   r<  r   rM     s<    
	zWorkersTransferBytes.updateN)r   r   r   r   r   r   r0  z  s   Br0  c                   @  s$   e Zd ZdZedd Zdd ZdS )Hardwarerl   c                   s  | _ tg g d _tf dddtg dd| _ jjddd	 jd
 tddgd} j| t	dd jj
d _d jj_tg g d _tf dddtg dd| _ jjddd	 jd
 tddgd} j| t	dd jj
d _d jj_tg g d _tf dddtg dd| _ jjddd	 jd
 tddgd} j| t	dd jj
d _d jj_t j j j _g g d _g g d _g g d _ fdd} j j| d S )N)size	bandwidthzDisk Bandwidth -- Computing ...rz   r{   )factorsr}   r~   r   r   rB  rC  r   ru   r   r   r   vline)Z	Bandwidthz@bandwidth{0.00 b}/s)moder   r   r   r   Fz!Memory Bandwidth -- Computing ...z"Network Bandwidth -- Computing ...c                    s    j  I d H } t| d tdD ]0}| d | } jd |  jd | q t| d tdD ]0}| d | } jd |  jd | qbt| d tdD ]0}| d | } jd |  jd | qd S )NdiskkeyrB  rC  r  network)r   Zbenchmark_hardwaresortedr@   	disk_datar   memory_datanetwork_data)r   rB  rC  r
  r   r   fP  s    zHardware.__init__.<locals>.f)r   r   disk_sourcer,   r   disk_figurevbarr   r   r   r   r  xgridr   memory_sourcememory_figurenetwork_sourcenetwork_figurer   r   rO  rN  rP  ZloopZadd_callback)r   r   r   r   rQ  r   r
  r   r     s        
    
    
zHardware.__init__c                 C  s   | j d r| jjjdkrd S | jd | jj_| j d | jj_| jd | j	j_t
| j| j  t
| j| j t
| j| j d| j	j_d| jj_d| jj_d S )NrB  zDisk BandwidthzMemory BandwidthzNetwork Bandwidth)rN  rS  r}   r   rP  rY  r   rD  rO  rW  rM   rR  rV  rX  r
  r   r   r   rM   d  s    

zHardware.updateN)r   r   r   r   rX   r   rM   r   r   r   r   rA    s   
vrA  c                   @  s,   e Zd ZdZedd Zeedd ZdS )BandwidthTypesz$Bar chart showing bandwidth per typec                 K  s   d| _ || _tddgddgddgddgd	| _tf d
ddddgd|| _d| jj_| jj| jdddddd}d| jj	_
tdd| jjd _tf t| jj_d |_d| jj_d| jj_d | j_t }d|_d|_| j| d S )Nr   r^   r_   r   rm   rn   12rC  bandwidth-halftypebandwidth_textzBandwidth by Typerz   Zbandwidth_type_histogram)r}   r~   r   y_range      r^  r_  rC  r   rr   r   ru   rv   r   r   rx   r   r   Fz@type: @bandwidth_text / sr   )r   r   r   r   r,   r   r   r  r   r   r   r   r  r   r   r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   y  sJ    	



zBandwidthTypes.__init__c                 C  sz   | j j}tt|| jj_t| dd | D t| dd | D d}dt	| j j
 | jj_t| j| d S )Nc                 S  s   g | ]}|d  qS r_   r   )r   rn   r   r   r   r     s     z)BandwidthTypes.update.<locals>.<listcomp>c                 S  s   g | ]}t |qS r   r<   r   ru   r   r   r   r     s     r]  Bandwidth: )r   Zbandwidth_typesr   rM  r   ra  rD  r   keysr<   rC  r}   r   rM   r   )r   bwr   r   r   r   rM     s    

zBandwidthTypes.updateNr   r   r   r   r   rZ  v  s   
+rZ  c                   @  s,   e Zd ZdZedd Zeedd ZdS )BandwidthWorkersr   c                 K  s  d| _ || _tddgddgddgddgd| _d	d
 tddD d d d }tddd
 |D ddd}tf dddddgddgd|| _t| jj	_
| jj| jdd|ddd |d | _t| jdd dd}tdd|_tf t|_| j|d d | j_t }d|_d |_| j| d S )!Nr   r^   r_   rm   rn   r[  r\  )rC  r   destinationr`  c                 S  s   g | ]}t |d d qS )r_   N)hexrg  r   r   r   r     s     z-BandwidthWorkers.__init__.<locals>.<listcomp>rd   rf   r   rC  c                 S  s   g | ]}d | | d qS )#ZFFr   rg  r   r   r   r     s     )
field_namepalettelowhighzBandwidth by Workerrz   Zbandwidth_worker_heatmap)r}   r~   r   r   ra  r   rl  )r   ru   rv   rx   r   r   Z	transform   )r   r   )Zcolor_mapperZlabel_standoffZborder_line_colorlocationr   r   r   z
            <div>
                <p><b>Source:</b> @source </p>
                <p><b>Destination:</b> @destination </p>
                <p><b>Bandwidth:</b> @bandwidth_text / s</p>
            </div>
            r   )r   r   r   r   r   r0   r,   r   r  r   r  r   	color_mapr   r   r  r   r   r  
add_layoutr   r   r   r   r   )r   r   r   r   ZmapperZ	color_barr   r   r   r   r     sb    	
	
zBandwidthWorkers.__init__c                   s   j j}|sd S fdd t fdd| D  \}}}t|j_ttt	|| }|j
j_|d d d j
j_|||ttt|d}dtj j j
j_tj| d S )Nc                   sB   z j j|  }W n tk
r(   |  Y S X |jd k	r>t|jS | S r  )r   r   KeyErrorr   r   )r   r   r
  r   r   r     s    


z%BandwidthWorkers.update.<locals>.namec                 3  s(   | ] \\}}} | ||fV  qd S r  r   )r   rm   rn   cr   r   r   r    s    
 z*BandwidthWorkers.update.<locals>.<genexpr>r   )r   rl  rC  r`  rh  )r   Zbandwidth_workersr%  r*  r  ru  rr  r   rM  setr   r   rD  ra  r9   r<   rC  r}   r   rM   r   )r   rj  ru   rv   r
   rD  r   r   )r   r   r   rM     s     	 
zBandwidthWorkers.updateNr   r   r   r   r   rk    s   
>rk  c                   @  s,   e Zd ZdZedd Zeedd ZdS )WorkerNetworkBandwidthzWorker network bandwidth chart

    Plots horizontal bars with the host_net_io.read_bps and host_net_io.write_bps worker
    state
    c              
   K  s  || _ tg g g g g g d| _tf dddd|| _| jjddd dd	d
d| jd | jjddd dd	dd| jd tf t| jjd _	t
dd| jjd _t| jj_d| jj_tdd| j_d| jj_d| jj_d | j_tf dddd|| _| jjddd dd	d
d| jd | jjddd dd	dd| jd tf t| jjd _	t
dd| jjd _t| jj_d| jj_tdd| j_d| jj_d| jj_d | j_d S )Ny_ready_writex_readx_writex_read_diskx_write_diskzWorker Network Bandwidthrz   Zworker_network_bandwidth)r}   r~   r   r}  r  r   r   rq   read)rv   r   r8  r   r   r9  legend_labelr   r~  r  rr   writer   r   r   FzWorkers DiskZworker_diskr  r  )r   r   r   r,   rC  r:  r   r   r   r  r   r   r  r  r  r   r!   r   r   r   r   r   rI  r   r   r   r   r     s    







zWorkerNetworkBandwidth.__init__c           
        sL  | j j }d  fddtt|D } fddtt|D }g }g }g }g }|D ]d}||jd d  ||jd d  ||jdi dd	 ||jdi dd	 qX| j jrtt|t|d
d| j	j
j | j	j
_tt|t|d
d| jj
j | jj
_nd
| j	j
_d
| jj
_||||||d}	t| j|	 d S )Nrp   c                   s   g | ]}|d  |   qS r;  r   r#  r<  r   r   r     s     z1WorkerNetworkBandwidth.update.<locals>.<listcomp>c                   s   g | ]}|d  |   qS r>  r   r#  r<  r   r   r     s     host_net_ioread_bps	write_bpshost_disk_ior    gffffff?r|  )r   r   r   r   r   r   r-  r   r  rC  r   r  rI  rM   r   )
r   r   r}  r~  r  r  r  r  r   r   r   r<  r   rM   }  sH    



	zWorkerNetworkBandwidth.updateNr   r   r   r   r   r{    s   
]r{  c                   @  s6   e Zd ZdZed
ddZdd Zeedd Zd	S )SystemTimeseriesa  Timeseries for worker network bandwidth, cpu, memory and disk.

    bandwidth
        Plots the average of host_net_io.read_bps and host_net_io.write_bps for the
        workers as a function of time
    cpu
        Plots the average of cpu for the workers as a function of time
    memory
        Plots the average of memory for the workers as a function of time
    disk
        Plots the average of host_disk_io.read_bps and host_disk_io.write_bps for the
        workers as a function of time

    The metrics plotted come from the aggregation of from ws.metrics[key] for ws in
    scheduler.workers.values() divided by number of workers.
     N  c              	   K  s.  || _ tg g g g g g g d| _t| j|   td|dd}d}tf dd||dd	|| _| jj| jd
dddd | jj| jd
dddd d| jj	_
d| jj_tdd| jjd _d| jj_d| jj_d| jj_tf dd||dd	|| _| jj| jd
dd d| jj_d| jj_d| jj_d| jj_tf dd||dd	|| _| jj| jd
dd d| jj_tdd| jjd _d| jj_d| jj_d| jj_tf d d||d!d	|| _| jj| jd
d"ddd | jj| jd
d#ddd d| jj	_
d| jj_tdd| jjd _d| jj_d| jj_d| jj_d S )$NrU   host_net_io.read_bpshost_net_io.write_bpscpur  host_disk_io.read_bpshost_disk_io.write_bpsr  r   followfollow_intervalrange_paddingzreset, xpan, xwheel_zoomz"Worker Network Bandwidth (average)r   z#worker_network_bandwidth-timeseries)r}   r   r~   r   r   rU   r  rq   zread (mean))r   ru   rv   rx   r  r  rr   zwrite (mean)top_leftzbytes / secondz0.0br   Fz Worker CPU Utilization (average)zworker_cpu-timeseriesr  )r   ru   rv   ZUtilizationzWorker Memory Use (average)zworker_memory-timeseriesr  ZByteszWorker Disk Bandwidth (average)zworker_disk-timeseriesr  r  )r   r   r   rM   get_datar   r,   rC  linelegendrt  r   
axis_labelr   r  ra  r   r   rU  r   r  r  rI  )r   r   r  r   r   r~   r   r   r   r     s      	




	



	



	



zSystemTimeseries.__init__c              	   C  s8  | j j }d}d}d}d}d}d}d}|D ]}	||	jd d 7 }||	jd d 7 }||	jd 7 }||	jd 7 }||	jdi dd7 }||	jdi dd7 }||	jd 7 }q,|t|pd	 d
 g|t|pd	 g|t|pd	 g|t|pd	 g|t|pd	 g|t|pd	 g|t|p*d	 gd}
|
S )Nr   r  r  r  r  r  r  rU   r^   r   r  )r   r   r   r-  r   r   )r   r   Znet_read_bpsZnet_write_bpsr  r  Zdisk_read_bpsZdisk_write_bpsrU   r   r   r   r   r   r  <  s2    
zSystemTimeseries.get_datac                 C  s   | j |  d | jjrptdd | jj D t| jj  }tdd | jj D t| jj  }nd}d}|d | jj	_
|| jj	_
d S )Nr   c                 s  s   | ]}|j pd V  qdS r^   Nnthreadsr   r   r   r   r  a  s    z*SystemTimeseries.update.<locals>.<genexpr>c                 s  s   | ]}|j V  qd S r  r  r   r   r   r   r  d  s    r^   r  d   )r   streamr  r   r   r   r   r   r  ra  r  r  )r   Z	y_end_cpuZ	y_end_memr   r   r   rM   [  s     

zSystemTimeseries.updateN)r  )	r   r   r   r   rX   r   r  r   rM   r   r   r   r   r    s   zr  c                   @  s,   e Zd ZdZedd Zeedd ZdS )ComputePerKey4Bar chart showing time spend in action by key prefixc           
      K  s  d| _ || _tj| jjkr,| jt| j ddgddgddgtd td	 gd
dgd}t|d| _t	f dddddgd|}|j
| jddddd}d|j_d|j_tdd|jd _tf t|j_t|j_d |_d|j_d|j_d |_t }d|_d|_|| | t!dd d!d" || _"t#|d#d$}t	f ddd%d&d|}|j$dd'd(t%d)d*d+t%d)d,dd| jd-	 d |j&_d|j&_d |j'_(| t!dd d!d" t }d|_d|_|| || _)t#|d.d$}	t*||	gd/d0| _+d S )1Nr   皙?rp   0.2 ms2.8 usgQ	@gQ?r?  computer   Zsum_partial)timesformatted_timeanglesrx   namesr   zCompute Time Per Taskrz   Zcompute_time_per_keyrm   rn   r}   r~   r   r   r  r  r   rx   r   ru   r   r   rx   Time (s)0r   F
            <div>
                <p><b>Name:</b> @names</p>
                <p><b>Time:</b> @formatted_time</p>
            </div>
            r   z1Note: tasks less than 2% of max are not displayeditalicr   Ztext_font_styleZbelowz	Bar Chartchildr}   zcompute_time_per_key-pie)rb  r   r^   r   r  T)Zinclude_zerowhite)	ru   rv   radiusZstart_angleZ	end_angler8  r9  legend_fieldr   z	Pie Chartstretch_both)tabssizing_mode),r   r   rR   r   plugins
add_plugints_color_lookupr   compute_sourcer,   rT  ra  r   r   r  r   r  r   r   r  r  r   r  r   r   rU  r   r   r   r   r   r   rv  r%   figrH   Zwedger.   r   Zgridgrid_line_colorZ	wedge_figr#   r   )
r   r   r   Zcompute_datar  r   r   tab1Zfig2tab2r   r   r   r   r  s    


zComputePerKey.__init__c                   s&  t t}| jj D ]<\}}t|}|j D ] \}}|dkr.||  |7  < q.qt| dd dd}|r"|d d d   fd	d
|D }t }t }t }	d|D ]2\}}|	| |	t
| |		| |7 qfdd
|	D }
|| jj_t|
|	||dd
 |	D d}t| j| d S )Nr  c                 S  s   | d S Nr^   r   ru   r   r   r   <lambda>      z&ComputePerKey.update.<locals>.<lambda>TrK  reverser   r^   g{Gz?c                   s    g | ]\}}| kr||fqS r   r   )r   nt)max_timer   r   r     s      z(ComputePerKey.update.<locals>.<listcomp>c                   s   g | ]}|  d  t j qS re  )r   pir   r  )
total_timer   r   r     s     c                 S  s   g | ]}t |qS r   r=   r  r   r   r   r     s     )r  r  rx   r  r  )r   r   r   task_prefixesr*  r?   all_durationsrM  r   r   ts_color_ofr  r   rD  dictrM   r  )r   Zcompute_timesrK  tsr   actionr  Zcompute_colorsZcompute_namescompute_timer  Zcompute_resultr   )r  r  r   rM     s:    



zComputePerKey.updateNr   r   r   r   r   r  o  s   
or  c                   @  s,   e Zd ZdZedd Zeedd ZdS )AggregateActionr  c                 K  s$  d| _ || _tj| jjkr,| jt| j ddgddgtd td gddgd}t|d	| _t	f d
ddddgd|| _
| j
j| jddddd}d| j
j_tdd| j
jd _d| j
j_tf t| j
j_t| j
j_d| j
j_d |_d| j
j_d| j
j_d | j
_t }d|_d|_ | j
!| d S )Nr   r  rp   r  r  r?  r  )r  r  rx   r  r  zAggregate Per Actionrz   Zaggregate_per_actionrm   rn   r  r  r  r   rx   r  r  r   r  Z16pxFr  r   )"r   r   rR   r   r  r  r  r   action_sourcer,   r   rT  ra  r   r   r   r  r  r   r   r  r  r   r  Zmajor_label_text_font_sizer   r   rU  r   r   r   r   r   r   )r   r   r   Zaction_datar   r   r   r   r   r     sN    





zAggregateAction.__init__c           	      C  s   t t}| jj D ](}|j D ]\}}||  |7  < q"qt| dd dd}t }t }t }|D ]>\}}|	| |dkr|	d n|	t
|  |	| qj|| jj_d| jj_t|||dd	 |D d
}t| j| d S )Nc                 S  s   | d S r  r   r  r   r   r   r  R  r  z(AggregateAction.update.<locals>.<lambda>Tr  r  ZpurplezAggregate Time Per Actionc                 S  s   g | ]}t |qS r   r  r  r   r   r   r   f  s     z*AggregateAction.update.<locals>.<listcomp>)r  rx   r  r  )r   r   r   r  r   r  r*  rM  r   r   r  r   r   rD  r}   r   r  rM   r  )	r   Z	agg_timesr  r  r  Z
agg_colorsZ	agg_namesZagg_timeZaction_resultr   r   r   rM   H  s.    


zAggregateAction.updateNr   r   r   r   r   r    s   
7r  c                   @  s,   e Zd ZdZedd Zeedd ZdS )MemoryByKeyz*Bar chart showing memory use by key prefixc                 K  s   d| _ || _tddgddgddgddgd	| _tf d
ddddgd|| _| jj| jddddd}tdd| jjd _	t
f t| jj_t| jj_d |_d| jj_d| jj_d | j_t }d|_d|_d|_| j| d S )Nr   rm   rn   r  r   r^   r_   rr   )r   r.  countrx   z
Memory Userz   Zmemory_by_keyr  r   r.  r   rx   r  r   r   Fz@name: @nbytes_textz
            <div>
                <p><b>Name:</b> @name</p>
                <p><b>Bytes:</b> @nbytes_text </p>
                <p><b>Count:</b> @count objects </p>
            </div>
            r   )r   r   r   r   r,   r   rT  r   r   r  r   r   r  r  r   r  r   r   r   r   r   r   r   r   r   rd  r   r   r   r   o  sH    	    


zMemoryByKey.__init__c                   s   t t t t| jj D ]<}|jD ]0}t|j} |  d7  < |  |j7  < q&qt	t
 }|| jj_| fdd|D fdd|D fdd|D dd |D d}dtt  | jj_t| j| d S )	Nr^   c                   s   g | ]} | qS r   r   r   r   )r   r   r   r     s     z&MemoryByKey.update.<locals>.<listcomp>c                   s   g | ]} | qS r   r   r  r.  r   r   r     s     c                   s   g | ]}t  | qS r   rf  r  r  r   r   r     s     c                 S  s   g | ]}t |qS r   rS   r  r   r   r   r     s     )r   r  r.  Znbytes_textrx   zTotal Use: )r   r   r   r   r   Zhas_whatr?   rK  r.  r   rM  r   r   rD  r<   r   r}   r   rM   r   )r   r   r  ksr  r   r   )r   r.  r   rM     s"    


zMemoryByKey.updateNr   r   r   r   r   r  l  s   
,r  c                   @  s.   e Zd ZdZedddZeedd ZdS )	CurrentLoadz"Tasks and CPU usage on each workerr   c           
   
   K  sH  d| _ || _tg g g g g g g g d| _tf dddt|d dd|}|j| jd	d
dddd}d|j_d |_	tf ddt|d dddd|}|j| jdd
dddd}d |_	||fD ]D}d|j
_d|j_d|j_ttddd}|| d |_d|j_qt }	d|	_d|	_||	 t }	d|	_d|	_||	 || _|| _d S )Nr   )nprocessingnprocessing-halfnprocessing-colorr  cpu-halfrv   rt   ry   zTasks Processingrz   r   r_   r|   )r}   r~   r   r   r   r  rv   r  r   r  rc  zCPU UtilizationZcpu_hist)r   r  )r}   r~   r   r   r   r   r  r  rr   Fr   r   r   z@worker : @nprocessing tasksr   z@worker : @cpu %)r   r   r   r   r,   r   r   r   r   r   r   r   r   r   r   r$   r   r   r   r   r   r   processing_figure
cpu_figure)
r   r   r   r   r   r   r  r  r   r   r   r   r   r     s    

	



zCurrentLoad.__init__c           	      C  s*  | j j }t }tdd |D s6|| jd k r6d S || _dd |D }dd |D }g }|D ]>}|| j jkr||d q`|| j jkr|d q`|d	 q`|d
d |D |dd |D |dd |D dd |D t	t
t|d}| j jrtdd |D }nd}|d | jj_t| j| d S )Nc                 s  s   | ]}|j V  qd S r  )r   r   r   r   r   r    s     z%CurrentLoad.update.<locals>.<genexpr>r^   c                 S  s   g | ]}t |jd  qS )r  )r   r-  r   r   r   r   r     s     z&CurrentLoad.update.<locals>.<listcomp>c                 S  s   g | ]}t |jqS r   r   r   r   r   r   r     s     rq   r   rr   c                 S  s   g | ]}|d  qS re  r   )r   rx  r   r   r   r     s     c                 S  s   g | ]}|d  qS re  r   )r   r   r   r   r   r     s     c                 S  s   g | ]
}|j qS r   r   r   r   r   r   r   !  s     c                 S  s   g | ]}t |jqS r   r   r   r   r   r   r   "  s     )r  r  r  r  r  rt   ry   rv   c                 s  s   | ]}|j pd V  qdS r  r  r   r   r   r   r  '  s     r  )r   r   r   rU   anyr   r   r   r   r   r   r   r  r  r   r  rM   r   )	r   r   nowr  r  Znprocessing_colorr   r   xranger   r   r   rM     s8     
zCurrentLoad.updateN)r   r   r   r   r   r   r    s   Mr  c                   @  s$   e Zd Zdd Zeedd ZdS )StealingTimeSeriesc                 K  s   || _ tt d t d d gddgddgd| _tdddd}tf dd	d
|d|| _| jj| jdddd | jj| jdddd d | jj_	| j
t tddtdd d S )Nr   r^   r   rU   r   r   r  r  r  z$Idle and Saturated Workers Over Timer   rz   r}   r   r~   r   rU   r   rq   )r   ru   rv   rx   r   r   r   Z
dimensions)r   r   rU   r   r   r,   r   r  r   Zminor_tick_line_colorr   r"   r    r'   )r   r   r   r   r   r   r   r   0  s0    
  zStealingTimeSeries.__init__c                   sV   t  d gtjjgtjjgd trDt  fdd nj	 d d S )Nr   r  c                     s   j  dS N'  r   r  r   r   r   r   r   r  T  r  z+StealingTimeSeries.update.<locals>.<lambda>r  )
rU   r   r   r   r   rK   r   add_next_tick_callbackr   r  r
  r   r  r   rM   K  s    
zStealingTimeSeries.updateNr   r   r   r   r   rX   rM   r   r   r   r   r  /  s   r  c                   @  s,   e Zd Zdd Zdd Zeedd ZdS )StealingEventsc              	   K  s   || _ |jd | _d| _tt d t gddgddgddgddgddgddgd| _td	d
dd}tf ddd|d|| _	| j	j
| jdddddd d| j	j_t }d|_d|_| j	|t tddtdd d S )Nstealingr   <      r  r^   r   )rU   levelrx   durationr  cost_factorr  r  r  r  zStealing Eventsr   rz   r  rU   r  rx   r  r   )r   ru   rv   rx   rB  r   ZLevelzLLevel: @level, Duration: @duration, Count: @count, Cost factor: @cost_factorr   r   r  )r   
extensionsstealr   r   rU   r   r   r,   r   circler   r  r   r   r   r   r"   r    r'   )r   r   r   r   r   r   r   r   r   Z  sP    
zStealingEvents.__init__c              	   C  s   d}|D ](}|dd \}}}}}}	}
}||7 }qzt | }W n ttfk
rZ   d}Y nX tt|dd d }|d |t||||| jj| d	}|S )
z Convert a log message to a glyphr   Nra   blackr      r_   r   )rU   r  r  rx   r  r  r  )	r+   rw  
IndexErrorr   sqrtr   r   r  Zcost_multipliers)r   ZmsgsZtotal_durationmsgrU   r  rK  r  satZocc_satZidlZocc_idlrx   r  r/  r   r   r   convert  s$    



zStealingEvents.convertc                   s   j jdd tj jd }|j } fddtd|d D  |_ rt ttdtt	j
ttjtttrt fdd njd d S )	Nr  )Ztopicc                   s4   g | ],} |  d  d dkr |  d  d  qS )r^   r   requestr   r#  logr   r   r     s      z)StealingEvents.update.<locals>.<listcomp>r^   c                     s   j  dS r  r  r   newr   r   r   r    r  z'StealingEvents.update.<locals>.<lambda>r  )r   Z
get_eventsr   eventsr   r   r5   r9   r8   r  r   r7   r  r   rL   rK   r   r  r   r  )r   r   r  r   r  r  r   r   rM     s$    

	zStealingEvents.updateN)r   r   r   r   r  r   rX   rM   r   r   r   r   r  Y  s
   /r  c                   @  s&   e Zd ZdddZeedd ZdS )Events   c              	   K  s   || _ t | _d| _|| _tg g g g g d| _tddd}tf |d|d|d|| _	| j	j
| jd	d
ddddd d| j	j_d| j	j_t }d|_d|_| j	|t tddtdd d S )Nr   rU   r  r   rv   rx   r  i@ )r  r  r   rz   )r}   r   r   r~   r   rU   rv   rx   r|   r   r  )r   ru   rv   rx   rB  r   r  Actionr  z@action<br>@hoverr   r   r  )r   r  	action_ysr   r   r   r   r   r,   r   r  r   r  r  rt  r   r   r   r   r"   r    r'   )r   r   r   r   r   r   r   r   r   r   r     sJ    		

zEvents.__init__c           
   	     s<  j jj  j jj j } rB fddtd|d D  j jj _ r8g }g }g }g }g } D ]\}}||d  |d }	||	 z|j|	  W n4 tk
r   t	jj|	< |j|	  Y nX |t
|	 |d qp|||||dtr*t fdd	 njd
 d S )Nc                   s   g | ]} |  qS r   r   r#  r  r   r   r     s     z!Events.update.<locals>.<listcomp>r^   r   r  ZTODOr  c                     s   j  dS r  r  r   r  r   r   r    r  zEvents.update.<locals>.<lambda>r  )r   r  r   Zevent_countsr   r   r   r  rw  r   rP   rK   r   r  r   r  )
r   r  actionsr  ZhoversysrT   Zmsg_timer   r  r   r	  r   rM     s>    
zEvents.updateN)r  r  r   r   r   r   r
    s   
+r
  c                   @  s&   e Zd ZdddZeedd ZdS )	
TaskStreamr   20sc                 K  s   || _ d| _tj| j jkr,| j t| j  | j jtj | _td| jj| | _t	 | _
|| _t|dd}|| _d| _d| _t|f|\| _| _dg| _d S )Nr   rw   default)r   offsetrR   r   r  r  pluginr  indexr  r   n_rectanglesrA   clear_intervalr   	last_seentask_stream_figurer   r   Ztask_stream_index)r   r   r  r  r   r   r   r   r     s    zTaskStream.__init__c                   s  j jj krd S j r\tjjd r\tjjd }tjjd }j| | d }nj}jjj j	|d t d }jj _  d sd S t
 jj krZt d j }tjjd }tttjjjd jjd }tjjd tj	 ||  }|| || d ks4|dk rZjjdd	  D  t d _fd
d d D  d< t
 _|dkrtr  D ](\}	}
t|
d trt|
 |	< qtrt  fdd nj j d S )Nr   r  r   )istartr   Zstart_boundaryr   r_   g?c                 S  s   i | ]
}|g qS r   r   r   r'  r   r   r   r)  Q  s      z%TaskStream.update.<locals>.<dictcomp>c                   s   g | ]}| j  qS r   )r  rg  r
  r   r   r   T  s     z%TaskStream.update.<locals>.<listcomp>r   r   c                     s   j  jS r  )r   r  r  r   
rectanglesr   r   r   r  _  r  z#TaskStream.update.<locals>.<lambda>)r  r  r   r   r   r   r  r  r  r   rU   r  r  r9   operatoraddr   rM   r   r*  
isinstancer   arrayrK   r   r  r  r  )r   r   r  boundaryr  Z	new_startZ	old_startZold_endZdensityr'  r(  r   r  r   rM   (  s\      


	 zTaskStream.updateN)r   r  r  r   r   r   r   r    s   
r  r  c           
      K  s*  t | dd} ttt |  gdgdgdgdgdgdgdgd	gd
gd
d}tdd}tdd}tf dd||dddddd	|}|j|dddddddddd
}d|_d|j_	d|j_
d|j_d|j_tdd d!}ttd"d#d$}|||t t td%d&td%d& tr"t }	|	| ||	 ||fS )'zG
    kwargs are applied to the bokeh.models.plots.Plot constructor
    rw   r  rp   r   r  z100 msZfoor   r^   ro   )
r   r  rK  r   rx   Zduration_textrt   rv   Zworker_threadr   r  r  task_streamzTask Streamr{   r   Nrz   r|   )	r   r}   r   ra  r   r   y_axis_locationr~   r   rv   r  r   rx   333333?r      )
r   ru   rv   r   r   r9  r8  
line_alphar   r  Fr   z
            <div>
                <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
                <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
            </div>
            r   z./profile?key=@namer   r   r   r  )rA   r   r  rU   r   r,   r   r   r   major_label_text_alphar   major_tick_line_alpharU  r   r   r$   r   r   r   r"   r    r'   rY   Zregister_plot)
r  r   r   r   ra  r   r   r   r   exportr   r   r   r  e  s    






r  c                   @  sJ   e Zd ZdZdd Zeedd ZedddZed	d
 Z	dd Z
dS )	TaskGraphz
    A dynamic node-link diagram for the task graph on the scheduler

    See also the GraphLayout diagnostic at
    distributed/diagnostics/graph_layout.py
    c              	   K  s  || _ t|| _|| j d| _tg g g g g g d| _tg g g d| _tddd}t	j
dk rnd|gi}nd	|i}tf |}tf |}t	j
dk r| j|_| j|_td
ddddddgddddddgd}tf ddi|| _tddd| _| j| jd | jjdd| jd |dd!d" | jjddd#|| j|d
d$}d | jj_d | jj_d%| jj_d%| jj_td&d'|gd(}	ttd)d*|gd+}
d |_| j|	|
 t !d,d-| _"d S ).Nr   )ru   rv   r   stater   rK  ru   rv   r   r   True)Zcolumn_namegroupr)  filtersfilterr/  waitingqueuedr   r  releasederredgrayZyellowr   rq   rr   r  )rD  rp  r}   z
Task Graph r  r  r{   ru   rv   r^   333333?)xsr  r   r  viewrx   r   r   )ru   rv   rB  rx   r   r=  r  Fr   z<b>@name</b>: @stater   r   	rendererszinfo/task/@key.htmlr   )r   r?  z%distributed.dashboard.graph-max-items  )#r   rN   layoutr  invisible_countr   node_sourceedge_sourcer   rJ   majorr   r   r/   r,   r   r%   subtitlerv  Z
multi_lineZsquarerU  r  r   r   r   r   r   r$   r   r   r   r;   r   	max_items)r   r   r   r4  Zfilter_kwargsZ	node_viewZ	edge_viewZnode_colorsr   r   r   r   r   r   r     sp    




		



zTaskGraph.__init__c                 C  s   t | jj| jkr@d| j_| j| jfD ]}dd |jD |_	q&n| j
t | jj	d d krp| j  d| _
d}nd}| jjg  }| j_| jj}g | j_| j|||d	 |   t | jjdkrd
| j_nd| j_d S )Nz(Scheduler has too many tasks to display.c                 S  s   i | ]
}|g qS r   r   )r   colr   r   r   r)  	  s      z$TaskGraph.update.<locals>.<dictcomp>ru   r_   r   TF)rM   Scheduler is empty.r:  )r   r   tasksrG  rF  r   rC  rD  Zcolumn_namesr   rB  rA  Zreset_indexr  	new_edgesadd_new_nodes_edgespatch_updates)r   	containerrM   r  rK  r   r   r   rM   	  s"    

zTaskGraph.updateFc              	   C  s  |s
|rg }g }g }g }g }g }	g }
| j j}| j j}| jj}|D ]x}z|| }W n tk
rj   Y qBY nX || }|| }|t| || || ||j	 ||j
j qB|D ]P\}}z0|	|| || g |
|| || g W q tk
r   Y qX q|||||dgt| d}|	|
dgt|	 d}|s\t| jjd sz| jj| | jj| n| j| | j| d S )Nr1  )ru   rv   r/  r   rK  r   r0  ru   )rA  ru   rv   r   rJ  rw  r   r:   r   r/  prefixr   r   rC  r   rM   rD  r  )r   r  rK  rM   Znode_keyZnode_xZnode_yZ
node_stateZ	node_nameZedge_xZedge_yru   rv   rJ  rK  ZtaskZxxyyrm   rn   nodeZedger   r   r   rL  $	  sT    



zTaskGraph.add_new_nodes_edgesc                   s   t | jjd t | jjd  | jjrZ| jj}g | j_fdd|D }| jd|i | jjr| jj}fdd|D }g | j_| jd|i |  jt |7  _| jj	r| jj	} fdd|D }g | j_	| jd|i dS )	zV
        Small updates like color changes or lost nodes from task transitions
        ru   c                   s    g | ]\}}| k r||fqS r   r   r   r  rx  r  r   r   r   d	  s      z+TaskGraph.patch_updates.<locals>.<listcomp>r/  c                   s    g | ]\}}| k r||fqS r   r   rR  rS  r   r   r   i	  s      r   c                   s    g | ]\}}| k r||fqS r   r   rR  )mr   r   r   p	  s      N)
r   rC  r   rD  rA  state_updatespatchZvisible_updatesrB  Zvisible_edge_updates)r   rU  Zupdatesr   )rT  r  r   rM  Y	  s$    zTaskGraph.patch_updatesc                 C  s   | j j| jjd d S )Nry  )r   Zremove_pluginrA  r   r
  r   r   r   __del__t	  s    zTaskGraph.__del__N)F)r   r   r   r   r   r   rX   rM   rL  rM  rW  r   r   r   r   r.    s   C4
r.  c                   @  s<   e Zd ZdZdd Zeedd Zdd Zedd	 Z	d
S )TaskGroupGrapha  
    Task Group Graph

    Creates a graph layout for TaskGroups on the scheduler.  It assigns
    (x, y) locations to all the TaskGroups and lays them out by according
    to their dependencies. The layout gets updated every time that new
    TaskGroups are added.

    Each task group node incodes information about task progress, memory,
    and output type into glyphs, as well as a hover tooltip with more detailed
    information on name, computation time, memory, and tasks status.
    c                 K  s  || _ i | _i | _d| _tg g g g g g g g g g g g g g g g g g g g g g g g g g d| _tg g g g d| _tf ddd|| _d| jj	_
tdd	d
| _| j| jd | jjdddddddd| jd	}| jjdddddd| jd | jjddddd d| jd  | jjdd!dddd d"| jd# ttd$d%dd&d'd(d)d*d+| jd,	| _| j| j d | jj_d | jj_d&| jj_d&| jj_td-d.|gd/}| j| d S )0Nr   )ru   rv   w_boxh_boxr   	tot_tasksrx   x_startr  y_starty_endx_end_progress	mem_alphanode_line_width
comp_tasksurl_logox_logoy_logow_logoh_logoin_processing	in_memoryin_releasedin_erredr  r  r<  r  xeyezTask Groups GraphT)r}   Zmatch_aspectFr:  r  r  r{   ru   rv   rY  rZ  rx   r`  r  ra  )	ru   rv   r   r   rx   r   r8  r  r   rc  rd  re  rf  rg  center)r   ru   rv   r  r=  anchorr   r\  r  r]  r^  )r   r   r   r   rx   r8  r   r_  r(  )r   r   r   r   rx   r8  r   r   ra   )rB  r   r^   r<  r  rm  rn  )	r  r8  r*  r  r\  r]  r  r^  r   r   a  
                <div>
                    <span style="font-size: 12px; font-weight: bold;">Name:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
                </div>
                <div>
                    <span style="font-size: 12px; font-weight: bold;">Compute time:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@compute_time</span>
                </div>
                <div>
                    <span style="font-size: 12px; font-weight: bold;">Memory:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
                </div>
                <div>
                    <span style="font-size: 12px; font-weight: bold;">Tasks:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@tot_tasks</span>
                </div>
                <div style="margin-left: 2em;">
                    <span style="font-size: 12px; font-weight: bold;">Completed:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@comp_tasks</span>
                </div>
                <div style="margin-left: 2em;">
                    <span style="font-size: 12px; font-weight: bold;">Processing:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_processing</span>
                </div>
                <div style="margin-left: 2em;">
                    <span style="font-size: 12px; font-weight: bold;">In memory:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_memory</span>
                </div>
                <div style="margin-left: 2em;">
                    <span style="font-size: 12px; font-weight: bold;">Erred:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_erred</span>
                </div>
                <div style="margin-left: 2em;">
                    <span style="font-size: 12px; font-weight: bold;">Released:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_released</span>
                </div>
                r>  )r   nodes_layoutarrows_layoutold_counterr   nodes_sourcearrows_sourcer,   r   r   r   r%   rF  rv  r   Z	image_urlr   r   r&   ZarrowsrU  r  r   r   r  ra  r   r   rd  r   r   r   r   	  s    




&+zTaskGroupGraph.__init__c                   s  dd | j j D }dd l}|jjdd | j j D |d}t| j j|jd}i  i t }i }i }|D ]
}|| rt fdd|| D d	 }	tfd
d|| D }
t	|| d	krt	fdd|| D d	kr|
d	7 }
nd}	rt
 d	 nd}
|	|
f|kr"|
d	7 }
q||	|
f |	|
  |< |<  | | d||< || |gt	||  d||< ql||fS )Nc                   s&   i | ]\ }  fd d|j D qS )c                   s   h | ]}|j  kr|j qS r   ry  )r   Zdsr'  r   r   r  '
  s     
 z:TaskGroupGraph.update_layout.<locals>.<dictcomp>.<setcomp>)dependencies)r   r  r   rv  r   r)  &
  s    z0TaskGroupGraph.update_layout.<locals>.<dictcomp>r   c                 S  s   i | ]\}}|j d qS )r^   ry  )r   r'  r2  r   r   r   r)  .
  s      )Zdskrw  rJ  c                 3  s   | ]} | V  qd S r  r   r   dep)r<  r   r   r  ;
  s     z/TaskGroupGraph.update_layout.<locals>.<genexpr>r^   c                 3  s   | ]} | V  qd S r  r   rx  r  r   r   r  <
  s     c                   s   h | ]} | qS r   r   rx  rz  r   r   r  ?
  s     z/TaskGroupGraph.update_layout.<locals>.<setcomp>)ru   rv   )nstartnend)r   task_groupsr*  r   orderrM  r   rz  r  r   r   r!  )r   rw  r   r~  Zordered	locationsrq  rr  tgru   rv   r   )r<  r  r   update_layout!
  sD    


zTaskGroupGraph.update_layoutc                 C  s(   d}d}|| ||  ||  | }|S )Nr   g?r   )r   ru   min_boxmax_boxr   r  rv   r   r   r   compute_sizeX
  s    zTaskGroupGraph.compute_sizec                   sv  j jjkrd S j j_j js.dj_ndj_j j j kr\ \__	g g g g g g g g g g g g g g g g g g g g g g g g g g d}g g g g d}t
 }t
 }j j D ](}|jr|jr||j ||j qt|dd}t|dd}t|dd}t|dd}	i  j j D ]\}
}|jd |jd  |jd	  }t|j }|jr|jr|rt|d
krt|d
krj|j| | || | || | d}j|j| | || | |	| | d}nd}|d }||d |
< q&j j D ]F\}
}j|
 d }j|
 d } |
 d } |
 d }|d | |d | |d | |d | |jd |jd  |jd	  }t|j }|d |jj |d t|jj |d | |d |jd t|j  d  |jd r&|d d n|d d
 |d ||d   |d ||d   |d ||d   |d  ||d  |d   |d! ||d  || |   |d"   fd#d$j	|
 d% D 7  < |d&  fd'd$j	|
 d% D 7  < |d(   fd)d$j	|
 d* D 7  < |d+  fd,d$j	|
 d* D 7  < t|jd
krtt|jd-d }zt| }W n tk
r   d.}Y nX nd.}|d/ | |d0 ||d1   |d2 ||d1   || }|d
kr&|d3 |d4  |d5 |d4 |  n(|d3 |d4 |  |d5 |d4  |d6 t |j |d t!|j |jd }|jd }|jd }|jd	 }|d7 | d8|| d9 d:d; |d< | d8|| d9 d:d; |d= | d8|| d9 d:d; |d> | d8|| d9 d:d; |d? | d8|| d9 d:d; qj"j#$| j%j#$| d S )@NrI  r:  )ru   rv   rY  rZ  r   rx   r[  r\  r  r]  r^  r_  r`  ra  rb  rc  rd  re  rf  rg  rh  ri  rj  rk  r  r  rl  r   r  r7  r  r8  r^   )r  r  r(  r_   r   r   ru   rv   r   r   rY  rZ  r   rx   r[  r`  r   r   ra     r\  r  r]  r^  r_  r<  c                   s*   g | ]"}j | d   | d d  qS ru   r   r_   rq  r  Zbox_dimr   r   r   r   
  s   z)TaskGroupGraph.update.<locals>.<listcomp>r{  r  c                   s   g | ]} j | d  qS rv   r  r  r
  r   r   r   
  s    rm  c                   s*   g | ]"}j | d   | d d  qS r  r  r  r  r   r   r   
  s   r|  rn  c                   s   g | ]} j | d  qS r  r  r  r
  r   r   r   
  s    .rz   rc  rd  r)  re  rg  r;  rf  r  rb  z (r  z.0fz %)rh  ri  rj  rk  )&r   transition_counterrs  r}  rF  r   rq  ri  r  rr  rz  r   r  Znbytes_totalr!  r   r  r*  Zstatesr   r   r  r   rO  r   rP   typesnextitersplit
logos_dictrw  r=   r<   rt  r   rM   ru  )r   Z
nodes_dataZarrows_dataZ	durationsr.  r  Zdurations_minZdurations_maxZ
nbytes_minZ
nbytes_maxrK  rb  r[  Z	width_boxZ
height_boxru   rv   r   r   Z	logo_typerc  ZratioZtasks_processingZtasks_memoryZtasks_relasedZtasks_erredr   r  r   rM   `
  s:   



	












zTaskGroupGraph.updateN)
r   r   r   r   r   r   rX   r  r  rM   r   r   r   r   rX  x	  s    5rX  c                   @  sN   e Zd ZdZdd ZddddZdddd	ZdddZee	dd Z
dS )TaskGroupProgressz3Stacked area chart showing task groups through timec                 K  s  || _ t | _d| _tj|jkr0|jt|d |jtj | _| j	t
| jjd d tdd}tdt| jj}tf ddd	d
||ddd d	|| _d| jj_d| jj_d| jj_d| jj_| jt t tddtdd d | _d | _t | _ |j!| _"t# | _$t# | _%d S )N   )r       @@rU   r   r%  zTask Group ProgressZtask_group_progressr{   r|   rz   r   )	r}   r   r   r   r   ra  r~   r   r'  Fr   r  )&r   r   r   nptsrO   r   r  r  r  r!  r   r#  rU   r   r!   r  r  r,   r   r   r+  r   r,  rU  r   r   r   r"   r    r'   _hover_last_drawn_offsetr  _last_transition_countr   
_renderers_line_renderers)r   r   r   r   ra  r   r   r   r   :  sL    





zTaskGroupProgress.__init__boolr  c                 C  sN   | j s
dS | j| jjks(| jj dkr,dS t| jj	 t| j
j	 k S )a7  
        Whether to add new renderers to the chart.

        When a new set of task groups enters the scheduler we'd like to start rendering
        them. But it can be expensive to add new glyps, so we do it deliberately,
        checking whether we have to do it and whether the scheduler seems busy.
        Tr|   F)r  r  r   r  proccpu_percentrz  r  r  ri  r   r   r
  r   r   r   _should_add_new_renderersj  s    	z+TaskGroupProgress._should_add_new_renderersc                 C  s   | j | jjko| jj dk S )z
        Whether to update the ColumnDataSource. This is cheaper than redrawing,
        but still not free, so we check whether we need it and whether the scheudler
        is busy.
        r|   )r  r   r  r  r  r
  r   r   r   _should_update  s    z TaskGroupProgress._should_updateFc                   sF  t tjjj dd tjjrtfddjj D }t|ttj	|dd }|7 ttj	|ddt| pd dkrjjd  njj jj
 }tjj  }tj||d	|r fd
djj D }nt fddjj}|j d |d< tjj  |d< |S )a!  
        Update the ColumnDataSource with our time series data.

        restrict_to_existing determines whether to add new task groups
        which might have been added since the last time we rendered.
        This is important as we want to add new stackers very deliberately.
        r   Nc                 3  s    | ]}t | d  V  qd S r  r   r#  r   r(  )frontr   r   r    s     z4TaskGroupProgress._get_timeseries.<locals>.<genexpr>rQ  )Ztrimrn   r^   )prependc                   s4   i | ],\}}|j jkr|t|   qS r   )r   r   r   r#  r&  backdtr  r   r   r   r)    s    z5TaskGroupProgress._get_timeseries.<locals>.<dictcomp>c                   s   t |    S r  r  r  )r  r  r  r   r   r    r  z3TaskGroupProgress._get_timeseries.<locals>.<lambda>r  rU   r  )r  r   r  rU   r  r  r   r   r   Z
trim_zerosr  r#  Zdiffr*  r6   r  r  )r   restrict_to_existingZaggZfront2r  
timestampsnew_datar   r  r   _get_timeseries  s4    	
z!TaskGroupProgress._get_timeseriesc              	   C  sP  |   r| jdd}|| j_t| jj}| jjj	|krB|| jj_	t
| jj }dd |D }tt||D ]\}\}}|| jkrt|| dksd| j| _d| j| _qnd| j| _d| j| _qn| jjdt|d|  t|d|d	   |d
| jd}|| j|< | jjdt|d|d	   |d| jd}	|	| j|< qn| jjr| jdkrtdd}
tddddd|
id| _| j| j | jrtdtt|| d| jid}
d}t| j D ]}|jr|} qؐq|g| j_d|
i| j_t  | _!| j"j#| _$nJ| % rLt| jj}| jjj	|kr2|| jj_	| jdd| j_| j"j#| _$dS )zy
        Maybe update the chart. This is somewhat expensive to draw, so we update
        it pretty defensively.
        F)r  c                 S  s   g | ]}t t|qS r   )rP   r?   r  r   r   r   r     s     z,TaskGroupProgress.update.<locals>.<listcomp>r   TrU   Nr^   r   )ru   y1y2rx   r   r   r   )ru   rv   rx   r   r   z
return '';)codeaj  
                        <div>
                          <div style="font-size: 1.2em; font-weight: bold">
                            <b>Worker thread occupancy</b>
                          </div>
                          <div>
                            $index{custom}
                          </div>
                        </div>
                        rG  ZnearestZ
horizontalz$index)r   rH  Zline_policy
attachment
formattersu  
                        const colormap = %s;
                        const divs = [];
                        for (let k of Object.keys(source.data)) {
                          const val = source.data[k][value];
                          const color = colormap[k];
                          if (k === "time" || k === "nthreads" || val < 1.e-3) {
                            continue;
                          }
                          const label = k.length >= 20 ? k.slice(0, 20) + '…' : k;

                          // Unshift so that the ordering of the labels is the same as
                          // the ordering of the stackers.
                          divs.unshift(
                            '<div>'
                              + '<span style="font-weight: bold; color:' + color + ';">'
                                + label
                              + '</span>'
                              + ': '
                              +  val.toFixed(1)
                              + '</div>'
                          )

                        }
                        divs.unshift(
                          '<div>'
                            + '<span style="font-weight: bold; color: darkgrey;">nthreads: </span>'
                            + source.data.nthreads[value]
                            + '</div>'
                        );
                        return divs.join('\n')
                        r   )r  args)&r  r  r   r   r  r  r  r   ra  r  r   r  ri  r  r%  r  r   Zcount_nonzeror   r  Zvarear1   r  r  r   r   r   r  reversedr   r?  r  rU   r  r   r  r  r  )r   r  Zmax_nthreadsZstackersrT   r  r2  rx   ZrendererZline_rendererr  Ztop_liner  r   r   r   rM     s    





 #'



zTaskGroupProgress.updateN)F)r   r   r   r   r   r  r  r  r   rX   rM   r   r   r   r   r  7  s   0
-r  c                   @  s(   e Zd ZdZdd Zeedd ZdS )TaskProgresszProgress bars per task typec                 K  s  || _ tti i i i i i d}t|d| _tdd}tdd}tf dd||d dd	d
|| _| jj	ddgddgddd | jj
| jddddddddd	 | jj
| jdddddddd | jj
| jdddddddd | jj
| jdddddd dd! | jj
| jdddd"d#d$dd! | jj
| jddd"d%d#d&d'd$dd(
 | jj
| jddd%d)d*d&dd$dd(
 | jj| jd+ddd,td-d. | jj| jd/ddd0dtd-d1 d2| jj_d| jj_d2| jj_d2| jj_d| jj_d2| jj_td3d4d5}| j| d S )6N)allr  r8  r7  r   r6  r  r   r%  iProgresstask_progressrz   r|   )r}   r   r   ra  r   r~   r   r   r   z#FFFFFFro   )ru   rv   r8  r   r   r   r   r   z#aaaaaarp   r;  )	r   r   r   r   r   r9  r8  r   r*  zreleased-locrx   r(  )r   r   r   r   r   r9  r8  r   z
memory-locr   z	erred-locr  r   )r   r   r   r   r   r9  r   r*  zprocessing-locr9  gffffff?z
queued-loc/r  )
r   r   r   r   r   r9  Zhatch_patternZhatch_colorr   r*  zno-worker-locrq   z	show-namer  Z10pt)r   r   rv   ru   x_offsettext_font_sizedone)r   r   rv   ru   r  Z
text_alignr  Fr   ay  
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">All:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Queued:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@queued</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">No-worker:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@no_worker</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Processing:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Memory:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Erred:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
                </div>
                r   )r   rQ   r  r   r   r   r!   r,   r   r  r   r   r
   r   r   r   r   rU  r   r   r   )r   r   r   r   r   ra  r   r   r   r   r   O  s    


   



	





!zTaskProgress.__init__c                   sT  i i i i i i i d| j j D ]}|j t fdd D r  d d |j<  d d |j<  d d |j<  d d |j<  d d |j<  d	 d	 |j<  d
 d |j< q fddd D d< d st| jj	d sd S t
}t| j| fdddD }|d tdd | D  |d< d| | jj_d S )N)r  r8  r7  r   r5  r6  	no_workerc                 3  s   | ]}  |V  qd S r  )r   )r   s)active_statesr   r   r    s     z&TaskProgress.update.<locals>.<genexpr>r  r8  r7  r   r5  r6  z	no-workerr  c                   s(   i | ]   t  fd d D qS )c                 3  s   | ]}|  V  qd S r  r   r  rv  r   r   r    s     z1TaskProgress.update.<locals>.<dictcomp>.<genexpr>r   r   )r   r/  rv  r   r)    s      z'TaskProgress.update.<locals>.<dictcomp>r  c                   s   i | ]}|t  |  qS r   r  r  r  r   r   r)    s    )r  r  r8  r7  r5  r6  r  c                 s  s   | ]\}}|d kr|V  qdS )r  Nr   r&  r   r   r   r    s     zProgress -- total: %(all)s, waiting: %(waiting)s, queued: %(queued)s, processing: %(processing)s, in-memory: %(memory)s, no-worker: %(no_worker)s, erred: %(erred)s)r   r  r   r  r  ri  r   r   r   r   rQ   rM   r   r*  r   r}   r   )r   tpr/  totalsr   )r  r/  r   rM     sB    

zTaskProgress.updateN)r   r   r   r   r   r   rX   rM   r   r   r   r   r  L  s    r  c                   @  s,   e Zd ZdZedd Zeedd ZdS )	EventLoopEvent Loop Healthc              	   K  s   || _ tddgddgddgd| _tf dddgdttjdd gd	d
d|| _| jj	ddd| jd d| jj
_d| jj_d| jj_tdgdd}| j| d S )NrV   Workersr   r  )r  r   r   r  zdistributed.admin.tick.interval   rz   r{   )r}   r   ra  r~   r   r  r   r   rF  TF)ZIntervalz@text srG  r   rH  )r   r   r   r,   rA   r   r;   r   r   rT  r   r   r   r   rU  r   r   )r   r   r   r   r   r   r   r   ,  s4    



zEventLoop.__init__c                 C  sb   | j }ddg|jtdd |j D t|jp2d gd}dd |d	 D |d
< t| j| d S )NrV   r  c                 s  s   | ]}|j d  V  qdS )Zevent_loop_intervalNr,  )r   r  r   r   r   r  U  s     z#EventLoop.update.<locals>.<genexpr>r^   )r  r   c                 S  s   g | ]}t |qS r   r  rg  r   r   r   r   Y  s     z$EventLoop.update.<locals>.<listcomp>r   r   )r   Z_tick_interval_observedr   r   r   r   rM   r   )r   r  r   r   r   r   rM   L  s    zEventLoop.updateNr   r   r   r   r   r  )  s   
r  c                   @  s:   e Zd ZU dZded< ddddddd	Zed
d ZdS )ExceptionsTablea  
    Exceptions logged in tasks.

    Since there might be many related exceptions (e.g., all tasks in a given
    task group fail for the same reason), we make a best-effort attempt to
    (1) aggregate to the task group, and (2) deduplicate similar looking tasks.
    rV   r   r   r   r   )r   r   r   c              
   K  s   || _ dddddg| _tdd | jD | _tdd	}tdd|d
dtdd|ddtdd|ddtdd|ddtddtddddg}d|krd|d i}ni }tf | j|dd|d dt|| _	d S )NTask	Exception	Traceback	Worker(s)Countc                 S  s   i | ]
}|g qS r   r   r  r   r   r   r)  t  s      z,ExceptionsTable.__init__.<locals>.<dictcomp>z.<code title="<%- value %>"><%= value %></code>)templater  )fieldr}   r  r   i,     z0,0r   r|   r  Tr   columnsZreorderableZsortabler   Zindex_position)
r   r  r   r   r   r)   r   r(   rI   r   )r   r   r   r   Zcode_formatterr  r  r   r   r   r   i  sp    !zExceptionsTable.__init__c                 C  s   dd | j D }| jj}|D ]^}|d |j |d |j |d |j |d d|j |d t	|j qt
| j| d S )	Nc                 S  s   i | ]
}|g qS r   r   r  r   r   r   r)    s      z*ExceptionsTable.update.<locals>.<dictcomp>r  r  r  r  z,
r  )r  r   erred_tasksr   rK  Zexception_textZtraceback_textjoinZerred_onr   rM   r   )r   r  r  r  r   r   r   rM     s    zExceptionsTable.updateN)r   )r   r   r   r   r   r   r   rM   r   r   r   r   r  ^  s
   
@r  c                   @  s8   e Zd ZdZdddddddhZdd
dZedd ZdS )WorkerTablezStatus of the current workers

    This is two plots, a text-based table for each host and a thin horizontal
    plot laying out hosts by their current memory use.
    Z	executingZ	in_flightri  readyrU   Zmanaged_bytesZspilled_bytes   c                   s4  |_ ddddddddd	d
dddddddg_j j }tfdd|D j _ddddddddd	d
ddddddg}dddddddddddd td d! jD _ fd"d!|D t	d#d$t	d%d$t	d&d$t	d&d$t	d&d$t	d&d$t	d&d$t	d&d$t	d'd$t	d'd$t	d(d$t	d(d$t	d'd$t	d'd$d)}t
f jfd*d+|D d,d,|d d-t}|D ]&}||krf|| |j|| _qfddgj }	 fd.d!|	D t
f jfd/d+|	D d,d,|d d-t}
|	D ]&}||kr|| |
j|	| _qtd0d1d2}tf d3d d4d5d6|d7d8d9|}|jjdd8d:d;d< d=|j_d8|j_d=|j_d=|j_||t  td0d>d2}tf d?d d4d5d6|d7d8d9|}|jjdd8d:d;d< d=|j_d8|j_d=|j_d=|j_||t  |_d@|krd@|d@ i}ni }|||g}jr$||
 t||_d S )ANr   r   r  r  r  r  memory_percentmemory_managedmemory_unmanaged_oldmemory_unmanaged_recentmemory_spillednum_fdsr  r  r  r  cpu_fractionc                   s>   h | ]6}|j  D ]&\}}| jkrt|tttfr|qqS r   )r-  r*  r  r"  r   r   r   )r   r   rT  r(  r
  r   r   r    s    
 z'WorkerTable.__init__.<locals>.<setcomp>r   zmemory %r   zunmanaged oldzunmanaged recentr   z# fdsznet readz	net writez	disk readz
disk write)r  r  r  r  r  r  r  r  r  r  r  c                 S  s   i | ]
}|g qS r   r   r  r   r   r   r)    s      z(WorkerTable.__init__.<locals>.<dictcomp>c              	     s"   i | ]}|t | ||d qS )r  r}   r)   r   r  column_title_renamesr   r   r)    s    z0 %r   z0.0 %r   z0 br  )r  r  r  r  r  r  r  r  r  r  r  r  r  r  c                   s   g | ]} | qS r   r   r   r  )r  r   r   r   &  s     z(WorkerTable.__init__.<locals>.<listcomp>Tr  c              	     s"   i | ]}|t | ||d qS r  r  r  r  r   r   r)  3  s    c                   s   g | ]} | qS r   r   r  )extra_columnsr   r   r   :  s     r   a  
                <div>
                  <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
                  <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent{0.0 %}</span>
                </div>
                r   zMemory Use (%))r   r^   )grp   r  rz   r   )r}   r   r   ra  r   r   r~   Zmin_border_rightr   r   )r   ru   rv   rB  r   Fa  
                <div>
                  <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu_fraction{0 %}</span>
                </div>
                zCPU Use (%)r  )r   r  r   r   rM  excluded_namesextra_namesr   r   r   r(   rI   r  r  r  r   r,   r  r   r   r   r   r   r   r   cpu_plotr   r   r   )r   r   r   r   r   Ztable_namesr  tabler   r  Zextra_tabler   Zmem_plotr  r  
componentsr   )r  r  r  r   r   r     sV   







	    
	    


zWorkerTable.__init__c              	   C  s  dd | j | j D }tt| jj dd dD ]V\}}|j}| j | j D ]P}d|kr|d\}}}|j	
|i 
|d }	n|j	
|d }	|| |	 qN|jd k	r|jn||d d< |j|d	 d< |jr|j	d
 |j |d d< nd|d d< |j|d d< |j|d d< |j|d d< |j|d d< |j|d d< |j|d d< |j	d d |d d< |j	d d |j |d d< |j|d d< q2| j | j D ]b}|dkr|| ddt||  d qzt| jjdkrd }
n|dkr6tdd | jj D }|r0tdd | jj D | nd}
n|dkrptdd | jj D d t| jj  }
nP|dkrtdd | jj D d tdd | jj D  }
nt|| }
|| d|
 W n& tk
r   || dd  Y nX q| jj| d S ) Nc                 S  s   i | ]
}|g qS r   r   r  r   r   r   r)    s      z&WorkerTable.update.<locals>.<dictcomp>c                 S  s
   t | jS r  )r   r   )r   r   r   r   r    r  z$WorkerTable.update.<locals>.<lambda>rJ  r  r   r   r   r  r  rz   r  r  r  r  r  r  g      Y@r  r  r   zTotal ()c                 s  s   | ]}|j V  qd S r  r  r   r   r   r   r    s    z%WorkerTable.update.<locals>.<genexpr>c                 s  s   | ]}|j d  V  qdS )r  Nr,  r   r   r   r   r    s   c                 s  s   | ]}|j d  V  qdS r  Nr,  r   r   r   r   r    s     r  c                 s  s   | ]}|j d  V  qdS r  r,  r   r   r   r   r    s     c                 s  s   | ]}|j V  qd S r  r  r   r   r   r   r    s     )r  r  r  rM  r   r   r   r  	partitionr-  r   r   r   r   r  r   r   r   r   r  insertr   r   	TypeErrorr   r   rM   )r   r   r  r   Zminfor   Zn0_Zn1r(  Z
total_dataZ	total_memr   r   r   rM     s    
 




zWorkerTable.updateN)r  )r   r   r   r   r  r   r   rM   r   r   r   r   r    s   
 Er  c                   @  s$   e Zd ZdZdd Zedd ZdS )	Shufflingrl   c                 K  s  t   || _tg g g g g g g g g g g g g g g g g g d| _tddddgddddgd| _tf dd	d
tddd|| _| jj| jddddd t	ddddgddidd}| j
| d| jj_d| jj_tdd| jjd _tf dd	d
tddd|| _d| jj_| jj| jdddd d t	d!d"d#d$gd%didd}| j
| tdd| jjd _tf d&d	d
d'|| _ddddg}tf |d(d d	d)|| _| jjd*d+d| jd, d | jj_d| jj_tdd| jjd _t	d-gd.d/}| j
| t| j| j| _W 5 Q R X d S )0N)rt   rv   comm_memorycomm_memory_limitZcomm_bucketsZcomm_avg_durationZcomm_avg_size	comm_readcomm_written
comm_colordisk_memorydisk_memory_limitZdisk_bucketsZdisk_avg_durationZdisk_avg_size	disk_readdisk_written
disk_colorNetwork SendNetwork Receive
Disk Write	Disk Readr   ru   r   zComms Bufferrz   r{   r  rE  r  rv   r   r  )r   r   rv   r   rx   )Memory Usedz@comm_memory{0.00 b})Average Writez@comm_avg_size{0.00 b})	# Bucketsz@comm_buckets)Average Duration@comm_avg_durationr  r   Zhline)r   r  rH  r^   r   r   zDisk BufferFr  r  )r   z@disk_memory{0.00 b})r  z@disk_avg_size{0.00 b})r  z@disk_buckets)r  @disk_avg_durationr  zTotal movement)r}   r~   r   ZTotals)r   r}   r   r~   ru   r   rF  )ZTotalz@values{0.00b}rG  r  )rX   r   r   r   totals_sourcer,   r!   r  r:  r   r   r   r   r  r   r   r  r  r   r   r  rT  rU  r  ra  r   r   )r   r   r   r   titlesr   r   r   r     s    









zShuffling.__init__c           
   	   C  s~  t  l | jjd j}|s*W 5 Q R  d S t| d }tt}t }t|	 D ]\}\}}|d 
| |d 
| dD ]f}|| d 
|| d  || d 
|| d	  || d
 
|| d  || d 
|| d  || d 
|| d dd || d 
|| d dd || d 
|| d  || d 
|| d  | jj| j|d k r|| d 
d q|| d	 || d kr|| d 
d q|| d 
d qqTdddd gt|d! t|d" t|d# t|d$ gd%}t| j| t| jt| t|d& |d'  d( }	|	| jj_|	| jj_W 5 Q R X d S ))Nshuffler   rv   rt   )ZcommrI  Z_totalr   Z_memoryr  Z_memory_limitr  Z_bucketsZbucketsZ_avg_durationZdiagnosticsZavg_durationr   Z	_avg_sizeZavg_size_readr  Z_writtenZwrittenr  Z_colorr9  rq   rr   r  r  r  r  r  r  r  r  r  r  r  g333333?)rX   r   r  Z
heartbeatsr   r   r   rU   r  r*  r   r   r   r  r   rM   r  r   r  r  r  r   r  r  )
r   inputr   r  r  rt   r/  rO  r  r   r   r   r   rM   \  sP    






	
zShuffling.updateN)r   r   r   r   r   r   rM   r   r   r   r   r    s   }r  z100%Z1920pxZ1080pxZ12pxz1px solid lightgrayzinset 1px 0 8px 0 lightgrayauto)r   r   z	max-widthz
max-heightpaddingZborderz
box-shadowZoverflowr)  styleZstylesc                   @  s   e Zd ZdddZdS )SchedulerLogsNc                 C  sL   |j |dd}|sd}ntddd |D  }tf d|it| _d S )NT)r   r  zC<p style="font-family: monospace; margin: 0;">No logs to report</p>
c                 s  s,   | ]$\}}}d t |d|f V  qdS )z%s - %sz%H:%M:%S.%fN)r   fromtimestampstrftime)r   rU   r  r  r   r   r   r    s   z)SchedulerLogs.__init__.<locals>.<genexpr>r   )Zget_logsrW   r  Z_repr_html_r*   _BOKEH_STYLES_KWARGSr   )r   r   r   ZlogsZ	logs_htmlr   r   r   r     s    
zSchedulerLogs.__init__)N)r   r   r   r   r   r   r   r   r    s   r  c                 C  sL   t | dd}d|_t||d ||j td|_|j	| t
|_d S )Nr  r  zDask: Scheduler System Monitorr   simple.html)rG   r}   rC   add_rootr   envget_templater  template_variablesrM   BOKEH_THEMEtheme)r   extradocZsysmonr   r   r   systemmonitor_doc  s    r  c              	   C  s   d|_ t| ddd}t| ddd}t| dddd}t| ddd}t||d t||d t||d t||d |jj|j_|	t
t|j|j|j|j|jtt
|j|j td	|_|j| t|_d S )
NzDask: Shufflingi  r  i@  r  i  )r   r   r  r   r  )r}   r  r  r  r  rC   rI  ra  rC  r  r   r   r   r  r  r  r  r  r  r  rM   r  r  )r   r  r  Z	shufflingr  Z
timeseriesZ
event_loopr   r   r   shuffling_doc  s<       r  c              	   C  s   t | }t| }t| }|jj|j_d|_t||d t||d t||d |t|jt	|j|jdd t
d|_|j| t|_d S )NzDask: Work Stealingr   r  r  r  )rk   r  r  r   r   r}   rC   r  r   r   r  r  r  r  rM   r  r  )r   r  r  rs   Zstealing_tsZstealing_eventsr   r   r   stealing_doc  s*    r  c                 C  s^   t | ddd}|  t||d d|_|t|jdd td|_	|j
| t|_d S )	Nr     )r   r   zDask: Scheduler EventsZscale_widthr  r  )r
  rM   rC   r}   r  r   r   r  r  r  r  r  r  )r   r  r  r  r   r   r   
events_doc  s    r!  c                 C  sP   t | }|  t||d d|_||j td|_|j	| t
|_d S )Nr   zDask: Exceptionsr  )r  rM   rC   r}   r  r   r  r  r  r  r  r  r   r  r  r  r   r   r   exceptions_doc"  s    r#  c                 C  sP   t | }|  t||d d|_||j td|_|j	| t
|_d S )Nr   zDask: Workersr  )r  rM   rC   r}   r  r   r  r  r  r  r  r  r"  r   r   r   workers_doc.  s    r$  c                 C  sP   t | }|  d|_||j td|_|j| t	|_
t||d d S )Nz Dask: Cluster Hardware Bandwidthr  r   )rA  rM   r}   r  r   r  r  r  r  r  r  rC   )r   r  r  Zhwr   r   r   hardware_doc:  s    r%  c                 C  s`   t | tjdddd}|  t||d d|_||j t	
d|_|j| t|_d S )Nz8distributed.scheduler.dashboard.tasks.task-stream-lengthZ60sr  r  r  r  r@  zDask: Task Streamr  )r  r   r;   r   rM   rC   r}   r  r   r  r  r  r  r  r  )r   r  r  r  r   r   r   	tasks_docG  s    r'  c                 C  sT   t | dd}d|_|  t||d ||j td|_|j	| t
|_d S )Nr  r  zDask: Task Graphr  r  )r.  r}   rM   rC   r  r   r  r  r  r  r  r  )r   r  r  graphr   r   r   	graph_docZ  s    r)  c                 C  sT   t | dd}d|_|  t||d ||j td|_|j	| t
|_d S )Nr  r  zDask: Task Groups Graphr  r  )rX  r}   rM   rC   r  r   r  r  r  r  r  r  )r   r  r  Ztg_graphr   r   r   tg_graph_docg  s    r*  c                 C  s  t | dd}|  t||d ||j t| jdkrZt| dd}t| dd}|j	}nt
| dd}t| dd}|j}t| dd}t| dd}t| dd}	|j}
|j}|  |	  |  |  |  t||d t||	d t||d t||d t||d ||j t|dd}t|
dd}t|dd}t|	jdd}t||||gd	dd
}|| t| tjdddd}|  t||d ||j t| dd}|  t||d ||j d|_t|_td|_|j| d S )Nr  r  r  
Processingr  ZCPUrk   zData TransferZprocessing_tabs)r  r   r  z9distributed.scheduler.dashboard.status.task-stream-lengthZ5sr&  zDask: Statuszstatus.html)r   rM   rC   r  r   r   r   r  r  r  r+  r   rk   r0  r  rH   r#   r  r   r;   r   r  r}   r  r  r  r  r  r  )r   r  r  r   r  r   Zprocessing_rootZcurrent_loadrs   r7  Zcpu_rootZoccupancy_rootr  r  Ztab3Ztab4Z	proc_tabsr&  r  r   r   r   
status_docs  sp    

r,  r   c              	   K  sb   t  R | |fddi|}|  t||| |t|| t|_dt|  |_W 5 Q R X d S )Nr  r  zDask: )	rX   rM   rC   r  r	  r  r  r>   r}   )clsintervalr   r  r  Zfig_attrr   r  r   r   r   individual_doc  s    r/  c                 C  s,   t | d|d}||j |  t|_d S Nr  r  r  )rF   r  r   trigger_updater  r  r   r  r  Zprofr   r   r   individual_profile_doc  s    r4  c                 C  s,   t | d|d}||j |  t|_d S r0  )rE   r  r   r2  r  r  r3  r   r   r   individual_profile_server_doc  s    r5  c                 C  sJ   d|_ t| d|d}||j td|_|j| t	|_
|  d S )NzDask: Profiler  r1  r  )r}   rF   r  r   r  r  r  r  rM   r  r  r2  r3  r   r   r   profile_doc  s    r6  c                 C  sJ   d|_ t| d|d}||j td|_|j| t	|_
|  d S )NzDask: Profile of Event Loopr  r1  r  )r}   rE   r  r   r  r  r  r  rM   r  r  r2  r3  r   r   r   profile_server_doc  s    r7  )r  )r   )
__future__r   loggingr   r   oscollectionsr   r   collections.abcr   r   Znumbersr   typingr   r	   ri   r   Zbokeh.core.propertiesr
   r   Zbokeh.ior   Zbokeh.layoutsr   r   Zbokeh.modelsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   Zbokeh.models.widgetsr(   r)   Zbokeh.models.widgets.markupsr*   Zbokeh.palettesr+   Zbokeh.plottingr,   Zbokeh.themesr-   Zbokeh.transformr.   r/   r0   r1   Zjinja2r2   r3   Ztlzr4   r5   r6   Ztlz.curriedr7   r8   r9   Ztornador:   r   r;   Z
dask.utilsr<   r=   r>   r?   r@   rA   Zdistributed.corerB   Z distributed.dashboard.componentsrC   Z'distributed.dashboard.components.sharedrD   rE   rF   rG   Zdistributed.dashboard.corerH   Zdistributed.dashboard.utilsrI   rJ   rK   rL   rM   Z$distributed.diagnostics.graph_layoutrN   Z distributed.diagnostics.progressrO   Z'distributed.diagnostics.progress_streamrP   rQ   Z#distributed.diagnostics.task_streamrR   r  rT   r  Zdistributed.metricsrU   Zdistributed.schedulerrV   Zdistributed.utilsrW   rX   r   Z!distributed.dashboard.export_toolrY   rZ   	getLoggerr   loggerpathr  dirname__file__r  r  r   r  r  r  rk   r   r   r   r  r+  r0  rA  rZ  rk  r{  r  r  r  r  r  r  r  r
  r  r  r.  rX  r  r  r  r  r  r  _STYLESrE  r  r  r  r  r  r!  r#  r$  r%  r'  r)  r*  r,  r/  r4  r5  r6  r7  r   r   r   r   <module>   s,  l 	
U)1  2p >d  A _Iz*dUS
T @   B   ^5Z  & N



"








J


