U
    /eD                     @   s  d dl Z e d d dlZd dlZd dlmZ d dlm	Z
 d dlmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ dd Ze j ddgddd Ze j ddgddd Z	e j ddgddd Ze!dddZ"dd Z#dd Z$dd Z%dd  Z&d!d" Z'd#d$ Z(d%d& Z)d'd( Z*d)d* Z+d+d, Z,d-d. Z-d/d0 Z.e j/0d1d2d3d4gd5d6 Z1d7d8 Z2e j/0d9ddgd:d; Z3e j/j4d<d=d>d? Z5d@dA Z6dBdC Z7dDdE Z8dFdG Z9dHdI Z:dJdK Z;e j/0dLddgdMdN Z<dS )O    Nnumpy)getitem)getter)getter_nofancy)_is_getter_task
fuse_sliceoptimizeoptimize_blockwiseoptimize_slices)	assert_eq)HighLevelGraph)SubgraphCallablefuse)SerializableLockc                 C   s"   |rt d| dfidddS | S dS )z
    Getters generated from a Blockwise layer might be wrapped in a SubgraphCallable.
    Make sure that the optimization functions can still work if that is the case.
    keyindex)ZoutkeyZinkeysN)r   )funcwrap r   F/tmp/pip-unpacked-wheel-dbjnr7gq/dask/array/tests/test_optimization.py_wrap_getter   s    r   TF)paramsc                 c   s   t t| jV  dS )z{
    Parameterized fixture for dask.array.core.getter both alone (False)
    and wrapped in a SubgraphCallable (True).
    N)r   	da_getterparamrequestr   r   r   r   $   s    r   c                 c   s   t t| jV  dS )z}
    Parameterized fixture for dask.array.chunk.getitem both alone (False)
    and wrapped in a SubgraphCallable (True).
    N)r   
da_getitemr   r   r   r   r   r   -   s    r   c                 c   s   t t| jV  dS )z
    Parameterized fixture for dask.array.chunk.getter_nofancy both alone (False)
    and wrapped in a SubgraphCallable (True).
    N)r   da_getter_nofancyr   r   r   r   r   r   6   s    r   )returnc                 C   s   t | dk st | t |kr dS t| d trHt| d j d d n| d }t|d trxt|d j d d n|d }||krdS t| dd |dd D ]4\}}tj	|rt
||s dS q||kr dS qdS )z
    Check that two tasks (possibly containing nested tasks) are equal, where
    equality is lax by allowing the callable in a SubgraphCallable to be the same
    as a non-wrapped task.
       Fr   NT)len
isinstancer   listdskvalueszipdaskcoreistask_check_get_task_eq)abZ
a_callableZ
b_callableZaeber   r   r   r)   ?   s    .."
r)   c                 C   sZ   |   |  kst|  D ]8\}}|| }tj|rHt||sTtq||kstqdS )au  
    Compare two getter dsks.

    TODO: this is here to support the fact that low-level array slice fusion needs to be
    able to introspect slicing tasks. But some slicing tasks (e.g. `from_array`) could
    be hidden within SubgraphCallables. This and _check_get_task_eq should be removed
    when high-level slicing lands, and replaced with basic equality checks.
    N)keysAssertionErroritemsr&   r'   r(   r)   )r*   r+   kavZbvr   r   r   _assert_getter_dsk_eqZ   s    	r2   c                 C   s,  | | dt ddft ddf| dt ddff|| dt ddt dd	fft ddt d
dff| dt ddt ddfff||dt ddt dd	fft ddt d
dff|dt ddt ddfff| | dt ddfdf| ddff|| dt dddfft ddff| dt dddfff||dt dddfft ddff|dt dddfff| | ddt ddfft ddff| ddt ddfff| | dt ddt dd	fft d d t d
dff| dt ddt ddfff| | dd t d d fft d d dff| ddff| | dt ddt ddfft ddff| dt ddt ddfff||dt ddfft ddt ddff|dt ddt ddfff| | dt ddddft ddf| dt ddff| | dt ddft ddddf| dt ddff| |dt ddddft ddddf|dt ddddffg}|D ]$\}}td|i}t|d|i qd S )Nx              d      2   <         
   i     NrA   i  Fy)slicer
   r2   )r   r   r   pairsinpexpectedresultr   r   r   test_fuse_getiteml   s    
   	"

	RrI   c           	   
   C   sV  t  }t  }| | dtddd|ftddf| dtddd|ff|| dtddtd	d
fd|ftddtddff| dtddtddfd|ff||dtddtd	d
fd|ftddtddff|dtddtddfd|ff| | dtddd|ftddd|f| | dtddd|ftddd|ffg}|D ]$\}}td|i}t|d|i q,d S )Nr3   r4   r5   Tr6   r7   r8   r9   r:   r;   r<   r=   r>   r?   rC   )r   rD   r
   r2   )	r   r   r   Zlock1Zlock2rE   rF   rG   rH   r   r   r   test_fuse_getitem_lock   sN    
-rJ   c                    s   d| dt ddt ddff| ddt d	d
ffd}t|dg}| ddt ddff t fdd| D sptt|t|k std S )Nz
some-arrayr*   r@   r7   r:   r;   r+   rA   r<   r=   )r*   r+   crK   r6   r>   r?   c                 3   s   | ]}t | V  qd S N)r)   .0vZexpected_taskr   r   	<genexpr>  s     z4test_optimize_with_getitem_fusion.<locals>.<genexpr>)rD   r   anyr$   r.   r    )r   r#   rH   r   rP   r   !test_optimize_with_getitem_fusion   s    rS   c              
   C   s   t df| dtd d d ff| dtd d d ff| dtddd ff| dtd d d ffd}d	| t dftddd ffi}tt|g d
dd }t|| | t dftdd d ff| dtddd ff| dtd d d ffd}tt|ddd	gd
dd }t|| d S )Nr@   r*   r+   rK   r   rA   d)r*   r+   rK   rT   erU   F)Zrename_keys)rK   rT   rU   )rangerD   r
   r   r2   )r   r#   rG   rH   r   r   r   test_optimize_slicing  s    
rW   c                	   C   s  t tddtdddtdddks(tt tddfd tddfd td	d
fksVtt tddftddd ftd	d
d fkstt dddkstt dtddfddksttt t tdddd W 5 Q R X tt t d tddg W 5 Q R X d S )Nr@   r6   r   rA      r:   r;   r7   n   x   )r   rL   )r   Nr   )NN   N)r   NN   N)r   rD   r.   pytestraisesNotImplementedErrornparrayr   r   r   r   test_fuse_slice   s    (

rc   c                   C   sR  t tddddddgdddgks&tt ddd	d
dgdddgd
dd	gksNtt ddd	d
dgdd
ksjtt ddd	d
dgddkstt ddd	d
dgtdd ddd
gkstt td tdddddgftd tddtd ftdd tdddddgfkstt td td dddgftd tdddftdd tdddfksNtd S )Nr@   r7   rX   r   r[               (   r<   r]   r   rA   )r   rD   r.   r   r   r   r   test_fuse_slice_with_lists;  s"    &(( 
 ri   c               
   C   s   t d } | dddg| fd| | ffd| | f| dddg| ff| ddg| | f| | | dffg}|D ](\}}tt t|| W 5 Q R X qZd S )Nr   rX   r[   r   )rD   r^   r_   r`   r   )nilZcasesr*   r+   r   r   r   test_nonfusible_fancy_indexingI  s    rk   c                 C   sD   d| | dd t d d fft d d dffi}tt|d| ddfi d S )Nr3   rA   rB   rD   r2   r
   )r   r#   r   r   r   test_hard_fuse_slice_casesX  s     "rm   c                  C   sV   t d} dD ]B}tj| dd}||j| }tdd | D dkst	qd S )Nr@   )rA   r@   ro   chunksc                 s   s   | ]}t |tjV  qd S rL   )r!   ra   ZndarrayrM   r   r   r   rQ   e  s     z.test_dont_fuse_numpy_arrays.<locals>.<genexpr>r   )
ra   onesda
from_array__dask_optimize__r&   __dask_keys__sumr$   r.   )r3   _rC   r#   r   r   r   test_dont_fuse_numpy_arrays_  s
    
ry   c                 C   s   t dd| dtddtddffd|dtd dffd}d	g}t||}t|d
ks^t| dd	h  }t	|| | dtdddffstd S )Nrf      r{   r3   r   r{   Zdxr   r   aliasr   r   )r3   r|   r}   Zdx2r   r   r[   )
ra   arangereshaperD   r   r    r.   r-   popr)   )r   r   r#   r-   dsk2Z	fused_keyr   r   r   test_fuse_slices_with_aliash  s    
r   c              	   C   sr   d| |dt ddd t ddd ffddgt d	d
d ffi}tt|| d| |ddddgfdfi}tt|| d S )Nr*   r3   r@   r7   r:   r;   r   r[   r<   r=   rX   r   rl   )r   r   r#   r   r   r   /test_dont_fuse_fancy_indexing_in_getter_nofancyv  s    r   rq   r@   rA   r[   c                    s   t dd  tjd| d} | }||j| }t fdd| D sTt| D ]<}t	|}|
d|
d dkst| k	r\d	|ks\tq\td
d | D }|jdkr||jkstn|dkstt| d  d S )Nr@   iIrp   c                 3   s   | ]}| kV  qd S rL   r   rM   r3   r   r   rQ     s     z0test_fuse_getter_with_asarray.<locals>.<genexpr>r   r   r   Z
1234567890c                 S   s   g | ]}t |r|qS r   )r   rM   r   r   r   
<listcomp>  s      z1test_fuse_getter_with_asarray.<locals>.<listcomp>r   )ra   rr   rs   ru   r&   rv   rR   r$   r.   strcountr    Znpartitionsr   )rq   rC   zr#   rO   sZ	n_gettersr   r   r   test_fuse_getter_with_asarray  s    
r   c                 C   sv   t dd }| d|ddfdf| | d|ddf|fdf| | d||fddfdfdfg}|D ]\}}ttd|id|i qRd S )Nr   r3   Fr   r*   rl   )r   nulloptsorigfinalr   r   r   $test_remove_no_op_slices_for_getitem  s    
r   whichc           	      C   s   | dkr|}n|}t dd }|d|ddf|d|ddff||d|ddf|f|d|ddff||d||fddfdf|d||fddffg}|D ]\}}ttd|id|i qd S )Nr   r   r3   Fr   r*   rl   )	r   r   r   r   getr   r   r   r   r   r   r   :test_dont_remove_no_op_slices_for_getter_or_getter_nofancy  s     	
r   z3blockwise fusion does not respect this, which is ok)reasonc               	   C   s   t jddd} t | d d d }||j| }tjddi ||j| }W 5 Q R X t|| t|| kst	t
|t
|k st	d S )	Nr@   rn   rp   r   rX   r[   zoptimization.fuse.ave-widthr   )rs   rr   rw   ru   r&   rv   configsetr   r.   r    )r3   rC   r*   r+   r   r   r   test_turn_off_fusion  s    $r   c               	   C   s   t jddi tjdddd} | j}|  }|||  }t|t	sLt
t|t	sZt
||ksft
|  } t|  t	st
t| dgd  W 5 Q R X dS )	zFCheck that by disabling fusion, the HLG survives through optimizationszoptimization.fuse.activeFr[   r[   int)rq   dtyper   N)r&   r   r   rs   rr   ru   __dask_graph__rv   r!   r   r.   Zpersistr   )rC   r   dsk1r   r   r   r   test_disable_lowlevel_fusion  s    r   c                  C   s~   t jddd} t jddd}t jdddd}| | | }| }t|dksPtt|}t|dkshtt|t	dd d	S )
z~
    Check that certain array creation routines work with blockwise and can be
    fused with other blockwise operations.
    r[   r   rp   rX   )Z
fill_valuerq   rA   r         @N)
rs   rr   zerosfullr   r    r.   r	   r   ra   )r3   rC   r   r*   r   r   r   r   r   $test_array_creation_blockwise_fusion  s    r   c                  C   sP   t dddgd} t | | d gf} | d}t tj|ddi}|  d S )Nr   rX   r   rX   r]   r   )rs   rt   ZconcatenateZrechunkZcoarsenra   rw   compute)r3   rC   r   r   r   test_gh3937  s
    
r   c                  C   sJ   t dd} tj| dd}|d }t||j}t|jdd| d S )N8   )      rz   rp   r   Foptimize_graph)	ra   r   r   rs   rt   dotTr   r   )r3   rT   Xr   r   r   test_double_dependencies  s
    r   c                  C   sh   t jddd} t jddd}| d d|d   }t|\}tttj|j dksZt	t
|| d S )Nr@   r   rp   r   rX   rA   )rs   rr   r   r&   r   rw   mapr(   r$   r.   r   )r3   rC   r   zzr   r   r   test_fuse_roots  s    r   c               	   C   s   t jddd} t jddd}tjdd |d }W 5 Q R X | d d|  }tj|j}t|jdkslt	d	did
d |j
 D kst	t ||j|j|j}t|| d S )Nr@   r   rp   bar)foorX   r   r[   r   c                 S   s   g | ]
}|j qS r   )annotations)rN   lr   r   r   r     s     z/test_fuse_roots_annotations.<locals>.<listcomp>)rs   rr   r   r&   ZannotateZ	blockwiser	   r    Zlayersr.   r$   Arraynamerq   r   r   )r3   rC   r   ZhlgZzar   r   r   test_fuse_roots_annotations  s     r   r   c                 C   s^   t jtddgddggdd}|d }t ||}|j| d}t|ddgddggsZtd S )Nr   rX   rp   r   rd      )rs   rt   ra   rb   matmulr   r   r.   )r   Zxxr   rH   r   r   r   ,test_optimize_blockwise_duplicate_dependency!  s
     r   )=r^   Zimportorskipr   ra   r&   Z
dask.arrayrb   rs   Zdask.array.chunkr   r   Zdask.array.corer   r   r   r   Zdask.array.optimizationr   r   r   r	   r
   Zdask.array.utilsr   Zdask.highlevelgraphr   Zdask.optimizationr   r   Z
dask.utilsr   r   Zfixtureboolr)   r2   rI   rJ   rS   rW   rc   ri   rk   rm   ry   r   r   markZparametrizer   r   r   Zxfailr   r   r   r   r   r   r   r   r   r   r   r   <module>   s^   



Y6	

 
	
