U
    /e9                     @   s   d dl mZ dZdd Zdd Zdd Zd	d
 Zdd Zd.ddZd/ddZ	d0ddZ
edddZdedfddZdd ZefddZdd Zd d! Zd1d"d#Zd2d$d%Zd&d' Zd(d) ZG d*d+ d+Zd,d- ZdS )3    )defaultdictZ__no_default__c                 C   s*   zt |  W dS  tk
r$   Y dS X dS )zpIs x hashable?

    Examples
    --------

    >>> ishashable(1)
    True
    >>> ishashable([1])
    False
    TFN)hash	TypeErrorx r   -/tmp/pip-unpacked-wheel-dbjnr7gq/dask/core.py
ishashable   s
    r	   c                 C   s   t | tko| ot| d S )zIs x a runnable task?

    A task is a tuple with a callable first argument

    Examples
    --------

    >>> inc = lambda x: x + 1
    >>> istask((inc, 1))
    True
    >>> istask(1)
    False
    r   )typetuplecallabler   r   r   r   istask   s    r   c                 C   s\   t |rdS z|| krW dS W n tk
r2   Y nX t|trX|D ]}t| |rB dS qBdS )zWhether ``x`` has anything to compute.

    Returns True if:
    - ``x`` is a task
    - ``x`` is a key in ``dsk``
    - ``x`` is a list that contains any tasks or keys
    TF)r   	Exception
isinstancelist	has_tasks)dskr   ir   r   r   r   )   s    


r   c                 c   sL   | D ]B}t |r t|E dH  qt|tr@tV  t|E dH  q|V  qdS )z(A generator to preorder-traverse a task.N)r   preorder_traversalr   r   )taskitemr   r   r   r   ?   s    
r   c                 C   s&   t |tr"tdd t| |D S | S )Nc                 s   s   | ]\}}t ||V  qd S N)lists_to_tuples).0rkr   r   r   	<genexpr>N   s     z"lists_to_tuples.<locals>.<genexpr>)r   r   r   zip)reskeysr   r   r   r   L   s    
r   Nc                    st   t | tr fdd| D S t| rP| d | dd  }}| fdd|D  S t| s\| S |  krl |  S | S dS )a  Do the actual work of collecting data and executing a function

    Examples
    --------

    >>> inc = lambda x: x + 1
    >>> add = lambda x, y: x + y
    >>> cache = {'x': 1, 'y': 2}

    Compute tasks against a cache
    >>> _execute_task((add, 'x', 1), cache)  # Compute task in naive manner
    2
    >>> _execute_task((add, (inc, 'x'), 1), cache)  # Support nested computation
    3

    Also grab data from cache
    >>> _execute_task('x', cache)
    1

    Support nested lists
    >>> list(_execute_task(['x', 'y'], cache))
    [1, 2]

    >>> list(map(list, _execute_task([['x', 'y'], ['y', 'x']], cache)))
    [[1, 2], [2, 1]]

    >>> _execute_task('foo', cache)  # Passes through on non-keys
    'foo'
    c                    s   g | ]}t | qS r   _execute_taskr   acacher   r   
<listcomp>q   s     z!_execute_task.<locals>.<listcomp>r      Nc                 3   s   | ]}t | V  qd S r   r    r"   r$   r   r   r   w   s     z _execute_task.<locals>.<genexpr>)r   r   r   r	   )argr%   r   funcargsr   r$   r   r!   R   s    
r!   c                 C   s   t |trt|n|gD ]}|| krt| dq|dkr@i }t| D ]}| | }t||}|||< qHt||}t |trt||}|S )zGet value from Dask

    Examples
    --------

    >>> inc = lambda x: x + 1
    >>> d = {'x': 1, 'y': (inc, 'x')}

    >>> get(d, 'x')
    1
    >>> get(d, 'y')
    2
    z is not a key in the graphN)r   r   flattenKeyErrortoposortr!   r   )r   outr%   r   keyr   resultr   r   r   get   s    




r1   Fc              	   C   s   g }|rg }|D ]}t |}|tkrH|rHt|d rH||dd  q|tkr\|| q|tkrt||  qz|| kr|| W q tk
r   Y qX q|}q|r|S t	|S )a  Returns the keys in `keys` that are also in `tasks`

    Examples
    --------
    >>> inc = lambda x: x + 1
    >>> add = lambda x, y: x + y
    >>> dsk = {'x': 1,
    ...        'y': (inc, 'x'),
    ...        'z': (add, 'x', 'y'),
    ...        'w': (inc, 'z'),
    ...        'a': (add, (inc, 'x'), 1)}

    >>> keys_in_tasks(dsk, ['x', 'y', 'j'])  # doctest: +SKIP
    {'x', 'y'}
    r   r'   N)
r
   r   r   extendr   dictvaluesappendr   set)r   tasksas_listretworkwtypr   r   r   keys_in_tasks   s$    r=   )returnc              	   C   s   t  }| rg }| D ]}t|}|tkrJ|rJt|d rJ||dd  q|tkr^|| q|tkrv||  qz|| W q t	k
r   Y qX q|} q|S )a  Returns all possible keys in `tasks` including hashable literals.

    The definition of a key in a Dask graph is any hashable object
    that is not a task. This function returns all such objects in
    `tasks` even if the object is in fact a literal.

    r   r'   N)
r6   r
   r   r   r2   r   r3   r4   addr   )r7   r9   r:   r;   r<   r   r   r   find_all_possible_keys   s"    r@   c                 C   s8   |dk	r| | }n|t k	r |}ntdt| |g|dS )a  Get the immediate tasks on which this task depends

    Examples
    --------
    >>> inc = lambda x: x + 1
    >>> add = lambda x, y: x + y
    >>> dsk = {'x': 1,
    ...        'y': (inc, 'x'),
    ...        'z': (add, 'x', 'y'),
    ...        'w': (inc, 'z'),
    ...        'a': (add, (inc, 'x'), 1)}

    >>> get_dependencies(dsk, 'x')
    set()

    >>> get_dependencies(dsk, 'y')
    {'x'}

    >>> get_dependencies(dsk, 'z')  # doctest: +SKIP
    {'x', 'y'}

    >>> get_dependencies(dsk, 'w')  # Only direct dependencies
    {'z'}

    >>> get_dependencies(dsk, 'a')  # Ignore non-keys
    {'x'}

    >>> get_dependencies(dsk, task=(inc, 'x'))  # provide tasks directly
    {'x'}
    NzProvide either key or task)r8   )
no_default
ValueErrorr=   )r   r/   r   r8   r(   r   r   r   get_dependencies   s    
rC   c                    s&    fdd   D }t|}||fS )aN  Get dependencies and dependents from dask dask graph

    >>> inc = lambda x: x + 1
    >>> dsk = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}
    >>> dependencies, dependents = get_deps(dsk)
    >>> dependencies
    {'a': set(), 'b': {'a'}, 'c': {'b'}}
    >>> dependents  # doctest: +SKIP
    {'a': {'b'}, 'b': {'c'}, 'c': set()}
    c                    s   i | ]\}}|t  |d qS ))r   rC   )r   r   vr   r   r   
<dictcomp>  s      zget_deps.<locals>.<dictcomp>)itemsreverse_dict)r   dependencies
dependentsr   rF   r   get_deps  s    rL   c                 c   sD   t | tr| V  n.| D ](}t ||r8t||dE dH  q|V  qdS )aJ  

    >>> list(flatten([1]))
    [1]

    >>> list(flatten([[1, 2], [1, 2]]))
    [1, 2, 1, 2]

    >>> list(flatten([[[1], [2]], [[1], [2]]]))
    [1, 2, 1, 2]

    >>> list(flatten(((1, 2), (1, 2)))) # Don't flatten tuples
    [(1, 2), (1, 2)]

    >>> list(flatten((1, 2, [3, 4]))) # support heterogeneous
    [1, 2, 3, 4]
    )	containerN)r   strr+   )seqrM   r   r   r   r   r+     s    

r+   c                 C   sJ   t t}tj}|  D ](\}}||  |D ]}||| | q*qd|_|S )z

    >>> a, b, c = 'abc'
    >>> d = {a: [b, c], b: [c]}
    >>> reverse_dict(d)  # doctest: +SKIP
    {'a': set([]), 'b': set(['a']}, 'c': set(['a', 'b'])}
    N)r   r6   r?   rH   default_factory)dr0   Z_addr   valsvalr   r   r   rI   3  s    rI   c              	      s   t | }|tkr | r t| d stz|t  kr<|  kr<W S W n tk
rR   Y nX |tkrp fdd| D S | S g } h}| dd D ]}t |}|tkr|rt|d rt| }nD|tkrڇ fdd|D }n&z||kr}W n tk
r   Y nX || q| dd t| S )zPerform a substitution on a task

    Examples
    --------
    >>> def inc(x):
    ...     return x + 1

    >>> subs((inc, 'x'), 'x', 1)  # doctest: +ELLIPSIS
    (<function inc at ...>, 1)
    r   c                    s   g | ]}t | qS r   subsr   r   r/   rS   r   r   r&   X  s     zsubs.<locals>.<listcomp>r'   Nc                    s   g | ]}t | qS r   rT   rV   rW   r   r   r&   a  s     )r
   r   r   r   r   rU   r   r5   )r   r/   rS   Z	type_taskZnewargsZhash_keyr(   Ztype_argr   rW   r   rU   E  s2    
rU   c                    s  |d kr}nt |ts|g}|s&g }t }t } d krLfddD  |D ]}||kr`qP|g}|rP|d }	|	|kr|  qf||	 g }
 |	 D  ]}||kr||kri }|d }|d |krt| || < qt| ||< t|t fddD }| g}|| ||d krV||d  }t||j	d}|| q |
  |rp|    S ddd	 |D }td
| |
| q|
r||
 qf|s||	 ||	 ||	 |  qfqP|rg S |S )Nc                    s   i | ]}|t  |qS r   rD   r   r   rF   r   r   rG     s      z_toposort.<locals>.<dictcomp>c                    s   i | ]}|  | qS r   )intersectionrX   )rJ   inplayr   r   rG     s      r   )r/   z->c                 s   s   | ]}t |V  qd S r   )rN   rV   r   r   r   r     s     z_toposort.<locals>.<genexpr>zCycle detected in Dask: %s)r   r   r6   popr?   lenrI   r5   min__getitem__reversejoinRuntimeErrorr2   remove)r   r   returncyclerJ   ZorderedZ	completedseenr/   ZnodescurZ
next_nodesZnxtZ
prioritiesprevrK   cycledepsr   )rJ   r   r[   r   	_toposortl  sn    








rj   c                 C   s   t | |dS )z:Return a list of keys of dask sorted in topological order.)rJ   rj   )r   rJ   r   r   r   r-     s    r-   c                 C   s   t | |ddS )ax  Return a list of nodes that form a cycle if Dask is not a DAG.

    Returns an empty list if no cycle is found.

    ``keys`` may be a single key or list of keys.

    Examples
    --------

    >>> inc = lambda x: x + 1
    >>> d = {'x': (inc, 'z'), 'y': (inc, 'x'), 'z': (inc, 'y')}
    >>> getcycle(d, 'x')
    ['x', 'z', 'y', 'x']

    See Also
    --------
    isdag
    T)r   rd   rk   rQ   r   r   r   r   getcycle  s    rm   c                 C   s   t | | S )an  Does Dask form a directed acyclic graph when calculating keys?

    ``keys`` may be a single key or list of keys.

    Examples
    --------

    >>> inc = lambda x: x + 1
    >>> inc = lambda x: x + 1
    >>> isdag({'x': 0, 'y': (inc, 'x')}, 'y')
    True
    >>> isdag({'x': (inc, 'y'), 'y': (inc, 'x')}, 'y')
    False

    See Also
    --------
    getcycle
    )rm   rl   r   r   r   isdag  s    rn   c                   @   s4   e Zd ZdZdZdd Zdd Zdd Zd	d
 ZdS )literalzBA small serializable object to wrap literal values without copyingdatac                 C   s
   || _ d S r   rp   )selfrq   r   r   r   __init__  s    zliteral.__init__c                 C   s   dt | jj S )Nzliteral<type=%s>)r
   rq   __name__rr   r   r   r   __repr__   s    zliteral.__repr__c                 C   s   t | jffS r   )ro   rq   ru   r   r   r   
__reduce__  s    zliteral.__reduce__c                 C   s   | j S r   rp   ru   r   r   r   __call__  s    zliteral.__call__N)	rt   
__module____qualname____doc__	__slots__rs   rv   rw   rx   r   r   r   r   ro     s   ro   c                 C   s.   t | s t| tks t| tkr*t| fS | S )a$  Ensure that this value remains this value in a dask graph

    Some values in dask graph take on special meaning. Sometimes we want to
    ensure that our data is not interpreted but remains literal.

    >>> add = lambda x, y: x + y
    >>> quote((add, 1, 2))
    (literal<type=tuple>,)
    )r   r
   r   r3   ro   r   r   r   r   quote
  s    
 
r}   )N)N)F)NFN)N)collectionsr   rA   r	   r   r   r   r   r!   r1   r=   r6   r@   rC   rL   r   r+   rI   rU   rj   r-   rm   rn   ro   r}   r   r   r   r   <module>   s*   
.

%)'
[
