U
    /e7                  
   @   s4  d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d d	lmZ ed
ZedZedZG dd dZd?eddddZdd Z e! Z"e! Z#dd Z$eddddZ%ej&'dddgdd Z(ej&)ddd  Z*ej&'dddgd!d" Z+ej&'dddgd#d$ Z,ej&)dd%d& Z-ej&'dddgd'd( Z.ej&)dej&'d)d*d+d*gd*d*id*hgd,d- Z/ej&'dddgd.d/ Z0ej&)dej&'d0eegd1d2 Z1ej&'d3d4d5d6d7d8d9d:gd;d< Z2d=d> Z3dS )@    N)add)delayed)	clone_key)	Blockwise)bind
checkpointchunksclonewait_on)HighLevelGraph)Tuple)import_or_nonez
dask.arrayzdask.dataframeZpandasc                   @   s   e Zd Zdd Zdd ZdS )NodeCounterc                 C   s
   d| _ d S )Nr   )n)self r   F/tmp/pip-unpacked-wheel-dbjnr7gq/dask/tests/test_graph_manipulation.py__init__   s    zNodeCounter.__init__c                 C   s$   t t d  |  jd7  _|S )Nd      )timesleeprandomr   )r   xr   r   r   f   s    zNodeCounter.fN)__name__
__module____qualname__r   r   r   r   r   r   r      s   r   )layersreturnc                C   sh  |   }|  }|d k	r|  }| |  | @ r>t| |  rRt| |  rft|rd|j |j  |j @ rt|j |j  |j @ rt|j |j  rt|j |j  rt|j |j  rt|j |j  rdtnP| | @ r*t|rd|j |j @ rJt|j |j @ rdtd S N)__dask_graph__keysAssertionErrorr   dependencies)abomitr   dsk1dsk2Zdskor   r   r   assert_no_common_keys    s*    
"r*   c              	   C   s|   |  j}|   j}| D ]Z\}}z|| }W n" tk
rR   |t|d }Y nX t|tr| rjt| rtqdS )zTest that all layers of the original collection exist in the cloned collection
    too and that Blockwise layers have not been materialized
    r   N)	r!   r   itemsKeyErrorr   
isinstancer   Zis_materializedr#   )ZclonedorigZolayersZclayerskvZcvr   r   r   assert_did_not_materialize;   s    


r1   c                  C   s   t  } tdttdi}t| jdtjddd	| jtjddd	| jt
jd	d
gd
d| jt
jd	gd	d| jt
jt| jdtj|d
d| jtj|d	d| jtj|d d
d| jtj|d d	d| jg
}d| _|| fS )Nr   
   zHello 1r2   r2      r   r   r      ZnpartitionszHello 2r   )r   pd	DataFramelistranger   r   daones
map_blocksdbfrom_sequencemapZItemZfrom_delayedddfrom_pandasmap_partitionsr   )cntdfcollsr   r   r   collections_with_node_countersP   s    rJ   z tuple[Tuple, Tuple, NodeCounter]c                 C   s   t  }tdtf|jdfdtf|jdfid|jdfidt t d}td|jdfid	|jd
fidt t d}| s| }| }t|t|t|t||fS )Nr%   r   r8   r&      r%   r&   c   dr4   rM   rO   )	r   r   h1r   h2setZto_dictr   r<   )r   rG   r(   r)   r   r   r   demo_tuplesj   s    .rT   r   FTc                 C   sD   t | \}}}t|d|gi}|jddd ks2t|jdks@td S )Nr   sync	schedulerr4   )rT   r   computer#   r   )r   t1t2rG   cpr   r   r   test_checkpoint|   s    r\   znot da or not ddc                  C   s0   t  \} }t|  }|jdd |jdks,td S )NrU   rV      )rJ   r   rX   r   r#   )rI   rG   r[   r   r   r   test_checkpoint_collections   s    
r^   c                 C   s<   t | \}}}t|}|jdddks*t|jdks8td S )NrU   rV   r   r8   rK   rK   )rT   r
   rX   r#   r   )r   rY   _rG   Zt1wr   r   r   test_wait_on_one   s    ra   c                 C   sR   t | \}}}t|d|gi}tj|ddidddgifks@t|jdksNtd S )Nr   rW   rU   r_   )rN   r4   r4   )rT   r
   daskrX   r#   r   )r   rY   rZ   rG   outr   r   r   test_wait_on_many   s    "rd   c                  C   s"  t  \} }tdd }t|  }||d   |jdks<t|d  | d  ksXttj|d | d  tj|d | d  t	j|d | d  t	j|d | d  t	j|d	 | d	  t
j|d
 | d
  t
j|d | d  t
j|d | d  t
j|d | d  d S )Nc                 S   s   d S r    r   r   r   r   r   r      s    z#test_wait_on_collections.<locals>.fr   r]   r   r8   rK   rN   r4            	   )rJ   r   r
   rX   r   r#   r>   utils	assert_eqrA   rD   )rI   rG   r   Zcolls2r   r   r   test_wait_on_collections   s     

rl   c              
   C   s  dt fddtfdi}dtdt fdtffi}ddd}| rntd|}t||dt dhdd}td|}n
|| t|dt fdtfg}t|dg}t|dg}t|d| d	}t|d| d	}t|d| d	}	t|d| d	}
t|| d
}t|| d
}t||d| d}|	 |	 kst
t||| d t||	| d t||
| d t||| d t||| d t|||| d t||||	|
|||dkst
td|gid||gi| d}t|ddgifkst
|d d }t|||| d t||| d d S )Nr%   r   r8   r&   rP   rL   )r$   rM   seedassume_layers)ro   r'   rn   ro   r   r'   r   )rK   rs   rs   rs   rs   rs   rs   rs   r   y)r'   ro   rs   r   )rQ   rR   r   r   Zfrom_collectionsrS   updater   r	   r!   r#   r*   rb   rX   )r   r(   r)   dsk3rY   rZ   t3Zc1c2c3Zc4Zc5Zc6Zc7rc   Zc8r   r   r   
test_clone   sD    
 
"rz   literalr   r6   c                 C   sD   t jddd}dd }t |d|d| d }t|}t||dd d S )	Nr2   r   r5   c                 S   s   | S r    r   )arrZlitr   r   r   noop   s    z0test_blockwise_clone_with_literals.<locals>.noopr   Trq   )r>   r?   Z	blockwiser	   r*   )r{   r|   r}   ZblkZclnr   r   r   "test_blockwise_clone_with_literals   s
    r~   c                 C   s  dt fddtfdi}dtdt fdtffi}ddi}t }dt f|jdfdtf|jdfi}d|jdfi}| rtd|idt i}t||d	t dhd	}t|||d
t dhdhd
}t||dt t d}n|| || || t|dg}t|dg}t|dt fdtfdg}	t	||	d| d}
t
ddd}|
 |t f d tj	ks^t|
 |tf d tj	ks~t|
 dkst|jdkstt	||	|d| d}t
ddd}| | d tj	kst| dkst|jdkstt	|	|d| d}t
ddd}t
ddd}| |t f d tj	ksDt| |tf d tj	ksdt| | d tj	kst| dkst|jdkstd S )Na-1r   r8   b-1c-1d-1erK   )r   r   )r   r   r   )r   r   rm   )rn   r   rs   rp   rf   r_   ri   )rQ   rR   r   r   r   r   rS   ru   r   r   r   r!   r   r#   rX   r   )r   r(   r)   rv   rG   Zdsk4Zdsk4brZ   rw   t4Zbound1Zcloned_a_nameZbound2Zcloned_c_nameZbound3Zcloned_d_nameZcloned_e_namer   r   r   	test_bind   sT      



    r   funcc                 C   s  t dd }|d}||}tjddd}|d }|j}tjddgdd}|d	d
 }|dd
 }	|	 }
t	dt
tdi}tj|dd}|dd
 }|dd
 }|d }| }t }| tkr tjddd|j}d|_t|||	|
|||f|||||fdd\}}}}}}}n0t|||	|
|||||||fdd	\}}}}}}}t|| t|| t||	 t||
 t|| t|| t|| t|||dd t|||dd t||	|dd t|||dd t|||dd t|||dd | | kst|jdks&| tks&ttj|| |jdksN| tksNttj||	 |jdksv| tksvttj||
 |jdks| tksttj|| |jdks| tksttj|| |jdks| tksttj|| |jdks| tkstd S )Nc                 S   s   | d S Nr8   r   re   r   r   r   double3  s    z+test_bind_clone_collections.<locals>.doubler8   r3   r4   r5   r   r9   c                 S   s   | d S r   r   re   r   r   r   <lambda>A      z-test_bind_clone_collections.<locals>.<lambda>c                 S   s   | d S Nr   r   re   r   r   r   r   B  r   r   r2   c                 S   s   | d S r   r   re   r   r   r   r   H  r   c                 S   s   | d S r   r   re   r   r   r   r   I  r   r   )childrenparentsr'   rn   )r'   rn   Trr   rN   rh      r]          $   )r   r>   r?   TrA   rB   rC   minr:   r;   r<   r=   rD   rE   rF   r   r   r@   r   r   r	   r1   r*   rX   r#   rj   rk   )r   r   d1Zd2Za1Za2a3Zb1Zb2Zb3Zb4rH   Zddf1Zddf2Zddf3Zddf4Zddf5rG   parentZd2cZa3cZb3cZb4cZddf3cZddf4cZddf5cr   r   r   test_bind_clone_collections0  s    










r   zsplit_every,nkeys)r8   i+  )rK      )rh      )Nr   )g333333 @r   )g    eA   )Fr   c           	      C   s   dd t dD }t|t|}t|| d}t| |ks@t|jddd ksTtt|| d}t| |d ksxt|jddt	t dkstddd	}t|t|}t
||| d
d}t| |d kst|jdddkstd S )Nc                 S   s   i | ]}d |f|qS )r%   r   ).0ir   r   r   
<dictcomp>  s      z$test_split_every.<locals>.<dictcomp>r   split_everyrU   rV   r   r8   )r&   rM   F)r   ro   )r   r8   )r=   r   r<   r   lenr!   r#   rX   r
   tupler   )	r   ZnkeysZdskrY   rM   rZ   rv   rw   r   r   r   r   test_split_every  s    
r   c               	   C   s   t dddddg} tt t| dd W 5 Q R X tt t| dd W 5 Q R X tt t| dd W 5 Q R X tt t| d	d W 5 Q R X tt t| ddid W 5 Q R X d S )
Nr   r8   rL   r%   r&   r   gffffff?r   )r   pytestZraises
ValueErrorr   	TypeError)tr   r   r   test_split_every_invalid  s    r   )N)4r   r   operatorr   r   rb   Zdask.bagZbagrA   r   Z	dask.baser   Zdask.blockwiser   Zdask.graph_manipulationr   r   r   r	   r
   Zdask.highlevelgraphr   Zdask.tests.test_baser   Zdask.utils_testr   r>   rD   r:   r   boolr*   r1   objectrQ   rR   rJ   rT   markZparametrizer\   Zskipifr^   ra   rd   rl   rz   r~   r   r   r   r   r   r   r   r   <module>   s~   








*


3
Q
