U
    /e1                     @  s   d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ eeZdddZG dd deZG dd deZdd ZG dd deZG dd deZdS )    )annotationsN)defaultdict)default_timer)ClassVar)groupbyvalmap)tokenize)	key_split	stringify)SchedulerPlugin)timeFc                 C  sv   t  }t  }t| }|rn| }|j}||kr0q|s<|jr<q|jdk	rV|| |sVq|| ||j q||fS )z
    All keys that need to compute for these keys to finish.

    If *complete* is false, omit tasks that are busy processing or
    have finished executing.
    N)	setlistpopkeyZwho_has	exceptionaddextendZdependencies)taskscompleteouterrorsstacktsr    r   D/tmp/pip-unpacked-wheel-g426oqom/distributed/diagnostics/progress.pydependent_keys   s"    



r   c                   @  s<   e Zd ZdZdddZdd	 Zd
d Zdd ZdddZdS )Progressa  Tracks progress of a set of keys or futures

    On creation we provide a set of keys or futures that interest us as well as
    a scheduler.  We traverse through the scheduler's dependencies to find all
    relevant keys on which our keys depend.  We then plug into the scheduler to
    learn when our keys become available in memory at which point we record
    their completion.

    State
    -----
    keys: set
        Set of keys that are not yet computed
    all_keys: set
        Set of all keys that we track

    This class performs no visualization.  However it is used by other classes,
    notably TextProgressBar and ProgressWidget, which do perform visualization.
    r   皙?FNc                 C  sx   |pdt |||| | _dd |D | _dd | jD | _|| _|| _|| _|| _d| _t | _	d| _
d | _i | _d S )Nz	progress-c                 S  s    h | ]}t |d r|jn|qS )r   )hasattrr   .0kr   r   r   	<setcomp>E   s     z$Progress.__init__.<locals>.<setcomp>c                 S  s   h | ]}t |qS r   )r
   r    r   r   r   r#   F   s     r   F)r   namekeys	schedulerr   Z_minimumZ_dtZlast_durationr   Z_start_timeZ_runningstatusextra)selfr%   r&   minimumdtr   r$   r   r   r   __init__C   s    zProgress.__init__c                   s    j }| jjs&tdI d H  q fdd|D }d  _  j  t| jd\ _	} jsr j	
  _ nt|dd\ _ } j	|   j | j	@ O  _  j s jd d d td |D ]} j|d dd	d
 qd S )N皙?c                   s   g | ]} j j| qS r   r&   r   r    r)   r   r   
<listcomp>W   s     z"Progress.setup.<locals>.<listcomp>r   Fr   r   Set up Progress keyserredTr   )r%   issubsetr&   r   asynciosleep
add_pluginr   r   all_keyscopyupdatestoploggerdebug
transitionr)   r%   r   r   _r"   r   r/   r   setupQ   s"    
zProgress.setupc                 O  s   || j kr@|dkr@|dkr@td| | j | | j s@|   || jkrn|dkrntd | j|d |d || j kr|dkrtd	| | jd
|d d S )N
processingmemoryzProgress sees key %sr4   Progress sees task erredr   r2   	forgotten,A task was cancelled (%s), stopping progressT)r%   r>   r?   remover=   r:   )r)   r   startfinishargskwargsr   r   r   r@   l   s    
zProgress.transitionc                 C  s   |    d S N)r=   r)   r&   r   r   r   restart|   s    zProgress.restartc                 C  sZ   | j | jjkr| jj| j d |rFd| _| j| jj| j|d nd| _t	
d d S )N)r$   errorr2   finishedzRemove Progress plugin)r$   r&   ZpluginsZremove_pluginr'   r(   r<   r   r   r>   r?   )r)   r   r   r   r   r   r=      s    zProgress.stop)r   r   FN)NN)	__name__
__module____qualname____doc__r,   rC   r@   rP   r=   r   r   r   r   r   /   s   
r   c                      s<   e Zd ZdZdedddf fdd	Zdd	 Zd
d Z  ZS )MultiProgressa  Progress variant that keeps track of different groups of keys

    See Progress for most details.  This only adds a function ``func=``
    that splits keys.  This defaults to ``key_split`` which aligns with naming
    conventions chosen in the dask project (tuples, hyphens, etc..)

    State
    -----
    keys: dict
        Maps group name to set of not-yet-complete keys for that group
    all_keys: dict
        Maps group name to set of all keys for that group

    Examples
    --------
    >>> split = lambda s: s.split('-')[0]
    >>> p = MultiProgress(['y-2'], func=split)  # doctest: +SKIP
    >>> p.keys   # doctest: +SKIP
    {'x': {'x-1', 'x-2', 'x-3'},
     'y': {'y-1', 'y-2'}}
    Nr   r   Fc                   s8   || _ dt||||| }t j||||||d d S )Nzmulti-progress-)r*   r+   r   r$   )funcr   superr,   )r)   r%   r&   rX   r*   r+   r   r$   	__class__r   r   r,      s         zMultiProgress.__init__c                   s2   j }| jjs&tdI d H  q fdd|D }d  _  j  t| jd\ _	} jsr j	
  _ nt|dd\ _ } j	|   j | j	@ O  _  j s jd d d ttt j j  _ ttt j j	 _	 j	D ]}| j krt  j |< q|D ]} j|d ddd	 q
td
 d S )Nr-   c                   s   g | ]} j j| qS r   r.   r    r/   r   r   r0      s     z'MultiProgress.setup.<locals>.<listcomp>r1   Fr2   r4   Tr5   r3   )r%   r6   r&   r   r7   r8   r9   r   r   r:   r;   r<   r=   r   r   r   rX   r@   r>   r?   rA   r   r/   r   rC      s,    

zMultiProgress.setupc                 O  s   |dkrV|dkrV| j | |d }|r:||kr:|| | j rNt| j  sV|   |dkrtd | |}|| j	kr|| j	| kr| j|d|d |dkr| |}|| j	kr|| j	| krtd| | jd	d
 d S )NrD   rE   r4   rF   r   r2   rG   rH   Tr5   )
r%   getrX   rI   anyvaluesr=   r>   r?   r:   )r)   r   rJ   rK   rL   rM   sr"   r   r   r   r@      s     



zMultiProgress.transition)	rS   rT   rU   rV   r	   r,   rC   r@   __classcell__r   r   rZ   r   rW      s       	!rW   c                 C  sd   t | d\}}t |d\}}|r<|dd|dd|ddS |rT|dd|ddS |ddS dS )zFormat seconds into a human readable form.

    >>> format_time(10.4)
    '10.4s'
    >>> format_time(1000.4)
    '16min 40.4s'
    >>> format_time(100000.4)
    '27hr 46min 40.4s'
    <   z2.0fzhr zmin z4.1fr_   N)divmod)tmr_   hr   r   r   format_time   s    
rf   c                   @  s,   e Zd ZdZdZdd Zdd Zdd Zd	S )
AllProgressz,Keep track of all keys, grouped by key_splitzall-progressc                 C  s   t t| _t dd | _t dd | _|| _| jj D ]V}|j}|j	j
}| j| | | j|j | | |jdkr8| j|  |j7  < q8||  d S )Nc                   S  s   dS )Nr   r   r   r   r   r   <lambda>       z&AllProgress.__init__.<locals>.<lambda>c                   S  s   t tS rN   )r   r   r   r   r   r   rh      ri   r   )r   r   allnbytesstater&   r   r^   r   prefixr$   r   r9   )r)   r&   r   r   rm   r   r   r   r,      s    

zAllProgress.__init__c           
      O  s  | j j| }|jj}| j| | z| j| | | W n tk
rP   Y nX |dkrx|j	dkrx| j	|  |j	8  < |dkr|j	dkr| j	|  |j	7  < |dkr| j| | | nL| j| }|| |s| j|= | j	
|d  | j D ]}	|	
|d  qd S )NrE   r   rG   )r&   r   rm   r$   rj   r   rl   rI   KeyErrorrk   r   r^   )
r)   r   rJ   rK   rL   rM   r   rm   r_   vr   r   r   r@     s(    

zAllProgress.transitionc                 C  s   | j   | j  d S rN   )rj   clearrl   rO   r   r   r   rP   &  s    
zAllProgress.restartN)rS   rT   rU   rV   r$   r,   r@   rP   r   r   r   r   rg      s
   rg   c                   @  s\   e Zd ZU dZdZded< ded< ded< ded	< d
d ZddddZdd Zdd Z	dS )GroupTimingzCKeep track of high-level timing information for task group progresszgroup-timingzClassVar[str]r$   zlist[float]r   zdict[str, list[float]]computenthreadsc                 C  s   || _ d| _|   d S )Ng      ?)r&   r+   _initrO   r   r   r   r,   3  s    zGroupTiming.__init__None)returnc                 C  s,   t  }|gd | _ i | _| jjgd | _dS )z7Shared initializatoin code between __init__ and restart   N)r   rr   r&   total_nthreadsrs   )r)   nowr   r   r   rt   <  s    zGroupTiming._initc                 O  s  |dkr|dkr| d}|s8td| d d S t }|| jd< | jd | jd  | jkr| jd | j | jd< | j| | j| jj | j	
 D ]}|d qqH| jj| }	|	j}
|
j| j	krdgt| j | j	|
j< |D ]}|d	 d
krq|d }|d }t| jd }|dkrD| j|d  |krD|d8 }q|dkr||kr|t| j|d  | }| j	|
j |  |7  < ||8 }|d8 }qDqd S )NrD   rE   
startstopszTask z@ finished processing, but timing information seems to be missingg        actionrr   r=   rJ      r   )r\   r>   warningr   r+   appendrs   r&   rx   rr   r^   r   groupr$   lenmax)r)   r   rJ   rK   rL   rM   rz   ry   gZtaskr   Z	startstopr=   idxdeltar   r   r   r@   H  s@    


zGroupTiming.transitionc                 C  s   |    d S rN   )rt   rO   r   r   r   rP   z  s    zGroupTiming.restartN)
rS   rT   rU   rV   r$   __annotations__r,   rt   r@   rP   r   r   r   r   rq   +  s   
	2rq   )F)
__future__r   r7   loggingcollectionsr   Ztimeitr   typingr   Ztlzr   r   Z	dask.baser   Z
dask.utilsr	   r
   Zdistributed.diagnostics.pluginr   Zdistributed.metricsr   	getLoggerrS   r>   r   r   rW   rf   rg   rq   r   r   r   r   <module>   s"   

]W4