U
    /e|                     @   s  d dl m Z  d dlZd dlZd dlZd dlmZ d dl	m
Z d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ dd Zdd Zdd Zdd Z dd Z!dd Z"dd Z#ej$%dddddgdd Z&d d! Z'd"d# Z(d$d% Z)d&d' Z*ej$%d(dej+ej,gd)d* Z-d+d, Z.d-d. Z/ej$%d/d0d1d2d2d2d2d2d2gd0d1d1d1d1d1d1d2gd0d0d0d0d0d1d1d2ggd3d4 Z0ej$j1d5d6 Z2d7d8 Z3d9d: Z4d;d< Z5ej$%d=d>d?gd@dA Z6dBdC Z7dDdE Z8dFdG Z9dHdI Z:dJdK Z;dLdM Z<dNdO Z=dPdQ Z>dRdS Z?dTdU Z@ej$%dVeAdWdXeAdYdZeAd[d\gd]d^ ZBd_d` ZCej$j1ej$%dadbdcgddde ZDdfdg ZEdhdi ZFdjdk ZGej$%dlddgd>gdmdn ZHdodp ZIdqdr ZJdsdt ZKdudv ZLdwdx ZMdydz ZNd{d| ZOd}d~ ZPdd ZQdd ZRej$%ddde ddde dddfgdd ZSdd ZTdd ZUdd ZVdd ZWdd ZXdd ZYej$%deZd0d1d2gdeY gdd Z[dd Z\ej$j1ej$%dddgdd Z]dS )    )datetimeN)config)	Blockwise)tm)_meta_from_array)optimize	assert_eq)Delayeddelayed)hlg_layer_topologicalc               	   C   s^  t jddgddggt jd} t| }t|tjs4t|d jt jksHt|d jt jks\tt	
|jtddg t jddgd	d
ggt jd} t| ddgd}t|tjst|d jt jkst|d jt jkstt	
|jtddg tt t| dddgd W 5 Q R X t jd t jdd} tj| dddgd} t| jdksZtd S )N            dtyper         ?       @      @      @abcolumnsc*      2   	chunksizer      )nparrayint64r   
isinstancepd	DataFrameAssertionErrorr   r   assert_index_equalr   Indexfloat64pytestraises
ValueErrorrandomseedZranddd
from_arraylen	divisionsxres r8   C/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/io/tests/test_io.pytest_meta_from_array   s$    r:   c               	   C   s  t jdddgt jd} t| }t|tjs.t|jt jks>tt jdddgt j	d} t| dd	}t|tjspt|j
dks~t|jt j	kstt jdddgt j	d} t| dgd	}t|tjst|d jt j	kstt|jtdg tt t| d
dgd	 W 5 Q R X d S )Nr   r   r   r   r   r   r   r6   r   r   r   )r"   r#   r+   r   r%   r&   Seriesr(   r   Zobject_namer'   r   r)   r   r*   r,   r-   r.   r5   r8   r8   r9   test_meta_from_1darray/   s     r=   c               	   C   s  t jdd tdD dt jfdt jfgd} t| }t|tjsDt	|d j
t jksXt	|d j
t jkslt	t|jtddg t| ddgd}t|tjst	|d j
t jkst	|d j
t jkst	t|jtddg tt t| dddgd W 5 Q R X d S )	Nc                 S   s   g | ]}||d  fqS 
   r8   .0ir8   r8   r9   
<listcomp>G   s     z+test_meta_from_recarray.<locals>.<listcomp>r?   r   r   r   r   r   )r"   r#   ranger+   r$   r   r%   r&   r'   r(   r   r   r)   r   r*   r,   r-   r.   r5   r8   r8   r9   test_meta_from_recarrayE   s      rE   c               	   C   s   t ddd} tj| dd}t|tjs0tt	|j
tdddg |jd	ksXt| j| k snttj| dtd
d}t|tjstt	|j
tdddg |jd	kst| j| k sttt tt jdd W 5 Q R X d S )N   r?   r   r   r    r   r   r   r   r      	   abcr   r   r   r   )r?   r?   r?   )shape)r"   arangeZreshaper1   r2   r%   r'   r(   r   r)   r   r&   r*   r4   computevaluesalllistr,   r-   r.   onesr6   dr8   r8   r9   test_from_arrayY   s    rU   c                  C   s   t jdd tdD ddgd} tj| dd}t|tjs>tt|j	d	d
gksTt|j
dksbt| jdd| k s~td S )Nc                 S   s   g | ]}||d  fqS r>   r8   r@   r8   r8   r9   rC   l   s     z5test_from_array_with_record_dtype.<locals>.<listcomp>r?   r   i4)r   rW   r   r   rG   r   r   rH   Findex)r"   r#   rD   r1   r2   r%   r'   r(   rQ   r   r4   rN   
to_recordsrP   rS   r8   r8   r9   !test_from_array_with_record_dtypek   s     r[   c               	   C   s  t d} tjt| tjt| dtjdt| dd}t	
|d}t|jdksVtt|jt|jd ksrtt|jd t|jd stt||  t	j
|d	d
}d}tt}t	j
|ddd W 5 Q R X |t|jksttttf}t	
| W 5 Q R X |t|jks"tt|jdks6tt|jt|jd ksTtt|jd t|jd sttt||  d S )NZaaaaaaabbbbbbbbccccccc)r   r   20120101startperiodsrX   r   r   r   rI   rG   z;Exactly one of npartitions and chunksize must be specified.r   )npartitionsr    )rQ   r&   r'   dictr"   r/   randnr3   
date_ranger1   from_pandasdaskr(   r4   r%   typerY   r   Zassert_frame_equalrN   r,   r-   r.   strvalue)r   dfddfmsgerrr8   r8   r9   test_from_pandas_dataframeu   s,     rm   c                  C   s  t ddddgi} dD ]}t| |}t| dks<t|jd dksNt|jd dks`ttj| |d}t| dkst|jd dkst|jd dkstqd	D ]l}d
D ]b}t ddg| i} tj| d|d}t| | t j	dg| dt
d}tj|d|d}t|| qqd S )Nr6   r   r   r   )r   r   rF   r   rG   )TF)r   r      r`   sort)r<   r   )r&   r'   r1   rd   r3   rN   r(   r4   r	   r;   int)ri   rB   r   rq   rj   sdsr8   r8   r9   test_from_pandas_small   s$    
ru   nr   r   r   ro   c                 C   sL   t jddddddgtddd	d
ddddgd}tj|| dj| ksHtd S )Nr   r   r   r   ro   r!   Zabdabdr6   yr?      rF   (   r   <   rX   r`   )r&   r'   rQ   r1   rd   r`   r(   )rv   ri   r8   r8   r9   (test_from_pandas_npartitions_is_accurate   s
     r}   c                  C   s   d} t jtj| t jd| dd}t|d}t|j	dksBt
t|jt|j	d ks^t
t|jd t|jd s|t
t||  tj|dd	}t|j	dkst
t|jt|j	d kst
t|jd t|jd st
t||  d S )
Nry   r\   r]   rX   r   r   r   rI   rG   )r&   r;   r"   r/   rb   rc   r1   rd   r3   re   r(   r4   r%   rf   rY   r   Zassert_series_equalrN   )rv   rs   rt   r8   r8   r9   test_from_pandas_series   s     r~   c                  C   sj   t jddddgidddgd} tj| ddd}|jr8tt| | tj| ddd}|jr\tt| | d S )	Nr6   r   r   r   rX   Frp   )r    rq   )r&   r'   r1   rd   known_divisionsr(   r	   ri   rj   r8   r8   r9   test_from_pandas_non_sorted   s    


r   c                  C   s@   t jddgidgd} tj| dd}|jdks2tt||  d S )Nr6   r   rX   r|   )r   r   )r&   r'   r1   rd   r4   r(   r	   r   r8   r8   r9   test_from_pandas_single_row   s    r   c                  C   sj   t ddddddddd	g	ttd
d} | jd| _t| d}t| | tj| dd}t| | d S )Nz
2015-08-28z
2015-08-27z
2015-08-26z
2015-08-25z
2015-08-24z
2015-08-21z
2015-08-20z
2015-08-19z
2015-08-18rJ   )DateZValzdatetime64[ns]r   rG   )	r&   r'   rQ   rD   r   Zastyper1   rd   r	   r   r8   r8   r9   $test_from_pandas_with_datetime_index   s&    

r   
null_valuec              	   C   sL   t jddddgid| dgd}tjtdd	 tj|dd
d W 5 Q R X d S )Nr6   r   r   r   CArX   z!is non-numeric and contains nullsmatchFrp   )r&   r'   r,   r-   NotImplementedErrorr1   rd   )r   ri   r8   r8   r9   !test_from_pandas_with_index_nulls   s    r   c               	   C   s   t jddddgidddgd} tjtdd td W 5 Q R X tjtd	d t|  W 5 Q R X tjtd
d tj| ddd W 5 Q R X tjtdd tj| dd W 5 Q R X d S )Nr6   r   r   r   rX   z$must be a pandas DataFrame or Seriesr   Zfooz2one of npartitions and chunksize must be specifiedzprovide npartitions as an intg@Frp   zprovide chunksize as an intgQE2@rG   )r&   r'   r,   r-   	TypeErrorr1   rd   r.   ri   r8   r8   r9    test_from_pandas_with_wrong_args   s     r   c                  C   sV   t jtjjddddtdd} tj| dd}t|t	
 }|dgd ksRtd S )	Nr   r?   r?   r   sizeABCDr   r   rG   )r&   r'   r"   r/   randintrQ   r1   rd   map_partitionsr3   rN   r(   )ri   rj   Znum_rowsr8   r8   r9   test_from_pandas_chunksize_one   s     r   rY   r   Br   c                 C   s:   t td| dd}tj|dd}|jdks6td S )NrI   r   rY   rY   r   r|   r   r   r   r   )r&   r'   rD   	set_indexr1   rd   r4   r(   )rY   ri   rj   r8   r8   r9   'test_from_pandas_npartitions_duplicates  s    	r   c                  C   sX   t d} ddddddddg}| td|dd}tj|dd	}|jd
ksTtd S )Ncudfr   r   r   rI   r   rY   r   r|   r   )	r,   importorskipr'   rD   r   r1   rd   r4   r(   )r   rY   ri   rj   r8   r8   r9   +test_gpu_from_pandas_npartitions_duplicates  s
    
r   c                  C   s   t jddd} tjtddddgd}t| dddg}t|jd	 rNt
t|| tj| dddgd}t|jd	 rt
t|| d S )	Nr?   r   r   r   chunksr   r   r   r   rn   )darR   r&   r'   r"   r1   from_dask_arrayr   re   is_materializedr(   r	   r2   )r6   pdfri   Zdf2r8   r8   r9   test_DataFrame_from_dask_array  s    
r   c                     sz   t jddd} | d9 } tjtdd dddgd}t| dddg t fd	d
tt	 j
jD sltt | d S )Nr   r   r   r   r   r   r   r   c                 3   s    | ]}t  j|  V  qd S N)r   re   r   r@   r   r8   r9   	<genexpr>3  s   zDtest_DataFrame_from_dask_array_with_blockwise_ops.<locals>.<genexpr>)r   rR   r&   r'   r"   r1   r   rP   rD   r3   re   layersr(   r	   )r6   r   r8   r   r9   1test_DataFrame_from_dask_array_with_blockwise_ops,  s    r   c                  C   st   t jddd} tjtddd}t| d}t|| ttd}t| }t|| t| }t|| d S )Nr?   r   r   r   r<   )	r   rR   r&   r;   r"   r1   r   r	   r2   )r6   ZpserserZser2r8   r8   r9   test_Series_from_dask_array:  s    



r   as_frameTFc                 C   sL   t jtjtdtdddd}| r,| }t j|j|j	d}t
|| d S )Nr?   Z
abcdefghijrX   r   r|   )r1   rd   r&   r;   rD   rQ   Zto_framer   rO   rY   r	   )r   rs   resultr8   r8   r9   test_from_dask_array_indexK  s
     r   c               	   C   s   t jjddd} tjtdd  tj| t	t
dd W 5 Q R X tjttdd	d
}tjttddd
}tjtdd tj|j|jd W 5 Q R X d S )Nr>   ro   )r   r   zmust be an instancer   r?   rX      r   r|   r   z".*index.*numbers of blocks.*4 != 2)r   r/   uniformr,   r-   r.   r1   r   r&   r*   r"   rM   rd   r;   rD   rO   rY   )r6   r   r   r8   r8   r9   !test_from_dask_array_index_raisesT  s    $r   c               	   C   sd   t jddd} td}tjtdd t|  W 5 Q R X tjtdd t| W 5 Q R X d S )N)r   r   r   r   r   zmore than 2D arrayr   )	r   rR   r"   r,   r-   r.   r1   r   r2   rw   r8   r8   r9   #test_from_array_raises_more_than_2D_  s    
r   c                  C   sN   t jddd} td}t| }t|}t|| t|}t|| d S )Nr   r   r   r   	r   rR   r"   r1   r   r&   r'   r	   r2   r6   rx   d1p1d2r8   r8   r9   'test_from_dask_array_compat_numpy_arrayj  s    




r   c               	   C   sp   t jddd} tjtdd tj| dgd W 5 Q R X td}tjtdd tj|dgd W 5 Q R X d S )Nr   r   r   znames must match widthr   r   r   )	r   rR   r,   r-   r.   r1   r   r"   r2   rw   r8   r8   r9   (test_from_array_wrong_column_shape_erroru  s    
r   c                  C   sl   t jddd} td}tj| dddgd}tj|dddgd}t|| tj|dddgd}t|| d S )Nr   r   r   r   r   r   r   r   r   r8   r8   r9   !test_from_array_with_column_names  s    

r   c                  C   sN   t jddd} td}t| }t|}t|| t|}t|| d S )Nr?   r   r   )	r   rR   r"   r1   r   r&   r;   r	   r2   r   r8   r8   r9   *test_from_dask_array_compat_numpy_array_1d  s    




r   c                  C   s^   t jddd} td}tj| dd}tj|dd}t|| tj| 	 dd}t|| d S )Nr?   r   r   r<   r   r   )
r   rR   r"   r1   r   r&   r;   r	   r2   rN   r   r8   r8   r9   $test_from_array_1d_with_column_names  s    

r   c                  C   s`   t jddd} td}tj| dgd}tj|dgd}t|| tj|dgd}t|| d S )Nr?   r   r   r<   r   r   r   r8   r8   r9   2test_from_array_1d_list_of_columns_gives_dataframe  s    

r   c                  C   s   t jddgddgd} tj| dd}t|}t|jt	
dd	g t|t	|  ttj|d	dgd
t	j| d	dgd
 d S )N)r   r   )r   r   rV   )r   objectr   )r   r   r   r   r   )r"   r#   r   r2   r1   r   r   r)   r   r&   r*   r	   r'   )r6   rx   ri   r8   r8   r9   !test_from_dask_array_struct_dtype  s    
 r   c                  C   s   t tdtddddtjtjfftdj} t| }t|tj	sPt
|jrZt
t|t	tddd tjdtjd	d
}t |dtjtjfdftj} t| }t|tjst
|jrt
t|t|  dd d S )Nro      ))r6   r   )r6   r   r6   r   F)Zcheck_indexr   r   ro   r   )r6   r   r   )r6   r   r   )r   )r   Arrayr"   rM   nanr   r1   r   r%   r;   r(   r   r	   r&   r/   r+   r'   rN   )dxri   dskr8   r8   r9   #test_from_dask_array_unknown_chunks  s     




r   zchunksizes, expected_divisions)r   r   r   r   )r   r   r   ro   ro   )r   r   r   r   )r   r   r   r   ro   )r   r   r   r   )r   r   r   r   ro   c                 C   s0   t jtd| d}t|}|j|ks,td S )Nr!   r   )r   r2   r"   rM   r1   r   r4   r(   )Z
chunksizesZexpected_divisionsZmonotonic_indexri   r8   r8   r9   !test_from_dask_array_empty_chunks  s    	
r   c               	   C   sd   t jdt jdd} t| dt jt jft jfft j}tjtdd t	
| W 5 Q R X d S )Nr   r   r   r6   z Shape along axis 1 must be knownr   )r"   r/   r   r   r   r+   r,   r-   r.   r1   r   )r   r   r8   r8   r9   (test_from_dask_array_unknown_width_error  s    "r   zarray_backend, df_backend)Zcupyr   )numpypandasc              	   C   s   t | }t |}td| i td}W 5 Q R X t|j|jsJt	t
|}t
| }t|j|jstt	t|j|jst	t|| d S )Nzarray.backendr?   )r,   r   r   setr   rR   r%   _metaZndarrayr(   r1   r2   rN   r;   r	   )Zarray_backendZ
df_backendZ	array_libZdf_libdarrZddf1ddf2r8   r8   r9   test_from_array_dispatching  s    


r   c                  C   s  t jddddgddddgd	t jd
dddgddd} t| d}|  t| dks`t	|d t| dkst	|jdd ddd	ddd	ddd	ddd	gkst	|jddd d
dddddddddddddddgkst	|j
d t| j
 kst	|j
  t| j
ks<t	|j
jddd ddiddiddiddigksrt	|j
jdd ddiddiddiddigkst	d S )Nr   r   r   rT   r   r   r   ro   rw   r   r   r   r   indr   rX   FTra   format)rY   r6   rx   r6   )r&   r'   r*   r1   rd   to_bagrN   rQ   Z
itertuplesr(   r6   items)r   rj   r8   r8   r9   test_to_bag  sB     





$r   c                  C   s   ddl m}  ddlm} tjtjddddgdd	d
dgdtjddddgddddd}|j	dd}t
||srtt| |j |jdf|jd  t| |j |jdf|jd  d S )Nr   )get)Bagr   r   r   rT   r   r   r   ro   rw   r   r   r   r   r   r   rX   r|   framer   r   )re   r   Zdask.bagr   r1   rd   r&   r'   r*   r   r%   r(   r	   r<   Z
partitions)r   r   rj   Zbagdfr8   r8   r9   test_to_bag_frame   s    r   c                  C   sr   t d ddlm}  tjddddgdd	d
dgdtjddddgddd}t|d}| |	 |	 dd d S )N
dask.arrayr   r   r   r   r   rT   r   r   r   ro   rw   r   r   r   r   r   r   rX   FZ
check_type)
r,   r   dask.array.utilsr	   r&   r'   r*   r1   rd   rZ   )r	   ri   rj   r8   r8   r9   test_to_records5  s    
  r   lengthsc                 C   s   t d ddlm} tjddddgdd	d
dgdtjddddgddd}t|d}|j	| d}||	 |dd t
|tjstd}|j|kstd S )Nr   r   r   r   r   r   rT   r   r   r   ro   rw   r   r   r   r   r   r   rX   r   Fr   ))r   r   )r,   r   r   r	   r&   r'   r*   r1   rd   rZ   r%   r   r   r(   r   )r   r	   ri   rj   r   Zexpected_chunksr8   r8   r9   test_to_records_with_lengthsD  s    
r   c               	   C   s   t d tjddddgdddd	gd
tjddddgddd} t| d}t t" |j	dddgd t 
d W 5 Q R X t t |j	d	d t 
d W 5 Q R X d S )Nr   r   r   r   rT   r   r   r   ro   rw   r   r   r   r   r   r   rX   r   z3 != 2zUnexpected value)r,   r   r&   r'   r*   r1   rd   r-   r.   rZ   Zfailr   r8   r8   r9   test_to_records_raisesY  s    
r   c            
   	      s  t jtjjddtdd} | jd d | jdd | jdd | jdd g  fd	d
tdD }|d  }dd }d dddddgfD ]}t	j
|||d}t||  t|| ddddgkst|j|d k	kstt	j
dd
 |D |j|d}t|| j t|| ddddgks*t|j|d k	kstqdd
 | jD }t	j
||d}t|jj|j tsvttt	j
||d|  tt	j
dd
 |D dd| j tt t	j
||ddddgd W 5 Q R X tt}	t	j
||jd  W 5 Q R X t|	jdstd S )Nr   r   abcddatar   r   r   r!   r?   c                    s   g | ]}t  j|qS r8   r   __getitem__r@   partsr8   r9   rC   m  s     z%test_from_delayed.<locals>.<listcomp>r   r   c                 S   s   t t| gS r   )r&   r;   r3   r6   r8   r8   r9   <lambda>p      z#test_from_delayed.<locals>.<lambda>)metar4   r   c                 S   s   g | ]
}|j qS r8   r   rA   rT   r8   r8   r9   rC   x  s     c                 S   s   g | ]}|d fqS )f8r8   )rA   r   r8   r8   r9   rC   }  s     r   c                 S   s   g | ]
}|j qS r8   r   r   r8   r8   r9   rC     s     )r   r   z)Metadata mismatch found in `from_delayed`)r&   r'   r"   r/   normalrQ   ilocrD   rN   r1   from_delayedr	   r   r(   r   r   r   r%   re   r   _namer   r,   r-   r.   rg   rh   
startswith)
ri   dfsr   Zmy_lenr4   rj   rs   Zmeta2Z	check_ddfer8   r   r9   test_from_delayedj  s0    4
"$ "r   c                  C   sn   t jttdd tdtddgd idd } t| jj	| j
 tsLttt| j|  j	dksjtd S )Nc                 S   s   t d| gd iS )Nr6   r?   r&   r'   r   r8   r8   r9   r     r   z3test_from_delayed_optimize_fusion.<locals>.<lambda>r?   r6   r   r   r   )r1   r   mapr   rD   r&   r'   r%   re   r   r   r   r(   r3   r   __dask_keys__)rj   r8   r8   r9   !test_from_delayed_optimize_fusion  s    r  c                  C   st   ddl m}  dd tdD }t|}| }dd | D }| |j|d}|| |	 }|j
dksptd S )	Nr   )optimize_blockwisec                 S   s    g | ]}t tjtd qS ))r   r   )r   r&   r'   r"   rR   r@   r8   r8   r9   rC     s     z3test_from_delayed_to_dask_array.<locals>.<listcomp>r   c                 S   s   g | ]}|d  qS )r   r8   )rA   kr8   r8   r9   rC     s     )keys)rJ   r   )dask.blockwiser  rD   r1   r   Zto_dask_arrayr  re   ZcullrN   rL   r(   )r  r   rj   Zarrr  r   r   r8   r8   r9   test_from_delayed_to_dask_array  s    

r  c                     s   t jtjjddtdd} | jd d | jdd | jdd | jdd g  fd	d
tdD }|d  }dd
 |D }t	j
||dj}|D ]F}|jj D ]4\}}|j| |kst|j| |jj| kstqqd S )Nr   r   r   r   r   r   r!   r?   c                    s   g | ]}t  j|qS r8   r   r@   r   r8   r9   rC     s     z4test_from_delayed_preserves_hlgs.<locals>.<listcomp>r   r   c                 S   s   g | ]
}|j qS r8   r   r   r8   r8   r9   rC     s     r   )r&   r'   r"   r/   r   rQ   r   rD   rN   r1   r   re   r   r   r(   Zdependencies)ri   r   r   ZchainedZhlgrT   Z
layer_nameZlayerr8   r   r9    test_from_delayed_preserves_hlgs  s    4r  c               	      s   t jdddddgdd tdD tdd	 t jdddddgtd
gd} tjt fdd g| d}tt}|	 j
dd W 5 Q R X d}|t|jkstd S )Nz(1)z(2)dateentvalc                 S   s"   g | ]}t |d  |d  d  qS r   )rD   r@   r8   r8   r9   rC     s     z5test_from_delayed_misordered_meta.<locals>.<listcomp>r   )r   r   rY   ro   )r   r   c                      s    S r   r8   r8   r   r8   r9   r     r   z3test_from_delayed_misordered_meta.<locals>.<lambda>r   sync)Z	schedulerzRThe columns in the computed data do not match the columns in the provided metadata)r&   r'   rD   r1   r   r   r,   r-   r.   Zreset_indexrN   rg   rh   r(   )Zmisordered_metarj   infork   r8   r   r9   !test_from_delayed_misordered_meta  s     r  c                  C   sj   t jdddgiddgd} t jdddgiddgd}tjt| t|gd	d
}|jsXt|jdksftd S )Nr6   r   r   r?   rX   r   d      sortedr4   )r   r  r  )r&   r'   r1   r   r   r   r(   r4   )r   r   r   r8   r8   r9   test_from_delayed_sorted  s
    
r  c                  C   s   t ddddgddddgd	} tj| dd
}| \}}t|tsHtt|tsVtt|	 | j
d d  |j }| }t|tstt|	 | d S )Nr   r   r   r   r?   ry   rF   rz   rw   r|   )r&   r'   r1   rd   
to_delayedr%   r
   r(   r	   rN   r   r6   sum)ri   rj   r   r   r6   r   r8   r8   r9   test_to_delayed  s     
r  c                  C   s   t dttdi} tj| dd}|d jd d }| d }t|j	dk sTt
|jddd }t|j	t|j	ks|t
t|d|  t|d|  |j }| }|jdd}t|j	t|j	k st
t| |  d S )	Nr6   ry   r|   r   r   r   F)Zoptimize_graph)r&   r'   rQ   rD   r1   rd   locr  r3   re   r(   r  r	   Zget_partitionrN   r6   r  )ri   rj   r   rT   r   r6   r   Zdx2r8   r8   r9   test_to_delayed_optimize_graph  s    
r  c                  C   s   t jddd} ttjddddttdd}tj|d	d
	d}tj
| |jdd}|jj|jjkslt|jj|jjkstttdddttdd}tj|d	d
	d}tj
| |jdd}|jj|jjkst|jj|jjkstd S )Nr>   r   r   z
2019-01-01r?   Z1T)r_   freq)r	  val1r   r|   r	  Zval2)rY   r   r   r   g?)idxr  r  )r   rR   r&   r'   rc   rQ   rD   r1   rd   r   r   rY   r   r(   r<   r"   rM   )r6   ri   rj   r   r8   r8   r9    test_from_dask_array_index_dtype
  s    
"r  valsr   r   )r   r   i  r?   i  r      c                 C   s   ddd}| d df| d dfg}t j||dd}tj| d | d | d | d gddddgd}|jj}t|dksxtt||j	 t
st|jt|kstt|| d S )	Nr   c                 S   s"   | \}}t j|g| |g| dS )NrX   r&   r;   )inputr   rh   rY   r8   r8   r9   func/  s    z"test_from_map_simple.<locals>.funcr   r   r   rX   )r   )r1   from_mapr&   r;   re   r   r3   r(   r%   r   r   r`   r	   )r  r"  iterabler   expectr   r8   r8   r9   test_from_map_simple#  s    

r&  c                  C   s   dd } t jdddt jdddgt jddgddt jddgddgf}t jddddgdd}tjdt jdd	dd	gddi|d
}tj| f| }t|| d S )Nc                 S   s   t d| | iS Naddr   rw   r8   r8   r9   r   I  r   z%test_from_map_multi.<locals>.<lambda>r   r$   r   r   r   r(  r   rX   r"   rM   r#   r&   r'   r1   r#  r	   )r"  	iterablesrY   r%  rj   r8   r8   r9   test_from_map_multiF  s    "$r+  c                  C   s   dd } t jdddt jdddg}t jddddgdd}tjdt jd	d
d	d
gddi|d}tj| |ddgd}t|| d S )Nc                 S   s   t d| | | iS r'  r   )r6   rx   zr8   r8   r9   r   X  r   z$test_from_map_args.<locals>.<lambda>r   r$   r   r   r   r(  ro   r!   rX   r   )argsr)  )r"  r$  rY   r%  rj   r8   r8   r9   test_from_map_argsU  s    $r.  c                  C   s^   dd } ddg}d}t j| ||d}tjddddgd	d
ddgd}|j|ksPtt|| d S )Nc                 S   s*   t j| d gd t| d | d d dS )Nr   r   r   rX   )r&   r;   rD   r   r8   r8   r9   r   d  r   z)test_from_map_divisions.<locals>.<lambda>)r   r   )r   r   )r   r   r   r  r   r   r   r   r   r   rX   )r1   r#  r&   r;   r4   r(   r	   )r"  r$  r4   r   r%  r8   r8   r9   test_from_map_divisionsa  s    

r/  c               	   C   s  ddd} ddg}t jdddddgiddddgd}t ddgijd d }tj| ||d	d
}t|j| t|| t ddgijd d }tj| ||d	d
}t|j| tjt	dd t|
 | W 5 Q R X tj| ||dd	d}t|j| t|
 | d S )Nr   c                 S   s   t d| g| iS )Nr6   r   )r6   rs   r8   r8   r9   r   u  r   z$test_from_map_meta.<locals>.<lambda>r   r   r6   r   rX   r   )r   rs   r   z The columns in the computed datar   F)r   enforce_metadatars   )r   )r&   r'   r   r1   r#  r	   r   r,   r-   r.   rN   )r"  r$  r%  r   rj   r8   r8   r9   test_from_map_metaq  s    
"
r1  c                  C   sp   dd } ddg}d}d}t jdddddgidd	dd	gd
}tj| |||d}|j|d | ksbtt|| d S )Nc                 S   s   t d| gd iS )Nr6   r   r   r   r8   r8   r9   r     r   z+test_from_map_custom_name.<locals>.<lambda>r   r   zmy-labelZ8675309r6   r   r   rX   )labeltoken-)r&   r'   r1   r#  r   r(   r	   )r"  r$  r2  r3  r%  rj   r8   r8   r9   test_from_map_custom_name  s    "r5  c                   c   s   t dddgE d H  d S )Nr   r   r   )	enumerater8   r8   r8   r9   
_generator  s    r7  r$  ))r   r   )r   r   )r   r   c                 C   sL   dd }t || }tjddddddgddddddgd	}t| | d S )
Nc                 S   s$   | d d }| d }t |g| S )Nr   r   r   )tr   r6   r8   r8   r9   r"    s    z+test_from_map_other_iterables.<locals>.funcr   r   r   r   r   r   rX   )r1   r#  r&   r;   r	   rN   )r$  r"  rj   r%  r8   r8   r9   test_from_map_other_iterables  s    r9  c                     s   g G  fddd t j  tdddgddd} tjddddddgd	gd
 dddddddgd}t| d |d  tdhkstt| | d S )Nc                       s.   e Zd ZdddZ fddZfddZdS )	z/test_from_map_column_projection.<locals>.MyFuncNc                 S   s
   || _ d S r   r   selfr   r8   r8   r9   __init__  s    z8test_from_map_column_projection.<locals>.MyFunc.__init__c                    s    |S r   r8   r:  )MyFuncr8   r9   project_columns  s    z?test_from_map_column_projection.<locals>.MyFunc.project_columnsc                    sT   |d d }|d }t |g| dg| d}| jd kr>|S  | j || j S )Nr   r   r?   r  )r&   r'   r   extend)r;  r8  r   r6   ri   )	projectedr8   r9   __call__  s    
z8test_from_map_column_projection.<locals>.MyFunc.__call__)N)__name__
__module____qualname__r<  r>  rA  r8   r=  r@  r8   r9   r=    s   
r=  r   r   r   ZmyfuncT)r2  r0  r?   r!   r  rX   r   )r1   r#  r6  r&   r'   r	   r   r(   )rj   r%  r8   rE  r9   test_from_map_column_projection  s"    rF  backendr   r   c              	   C   s|   t | }td| iZ ddddgdddd	gd
}||}tj|dd}t|| |j|dd}t|| W 5 Q R X d S )Nzdataframe.backendr   r   r   r   r?   r   r      )r   r   r|   )r,   r   r   r   r'   r1   	from_dictr	   )rG  Z_libr   expectedgotZgot_classmethodr8   r8   r9   test_from_dict_backends  s    


rL  )^r   r   r"   r   r&   r,   Z
dask.arrayr#   r   Zdask.dataframeZ	dataframer1   re   r   r  r   Zdask.dataframe._compatr   Zdask.dataframe.io.ior   Zdask.dataframe.optimizer   Zdask.dataframe.utilsr	   Zdask.delayedr
   r   Zdask.utils_testr   r:   r=   rE   rU   r[   rm   ru   markZparametrizer}   r~   r   r   r   ZNaTZNAr   r   r   r   Zgpur   r   r   r   r   r   r   r   r   r   r   r   r   r   r   paramr   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r&  r+  r.  r/  r1  r5  r7  r6  r9  rF  rL  r8   r8   r8   r9   <module>   s   





	





&
"


)